From f56098eed65ff1224447d391f8a1f82fa70d3fb0 Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Sun, 19 Jul 2026 16:08:37 +0200 Subject: [PATCH 1/3] fix(api): auto-recover Axelar stuck-confirm and stop leaked squidRouterPay polling loops Two fixes for the squidRouterPay phase, prompted by a Base->BNB transfer that sat in Axelar status "called" for 14h after its validator confirmation poll failed (Axelar's relayer never retries a failed poll): - Honor the phase processor's AbortSignal in the status polling loop and balance check. Previously every timed-out execution left an immortal 10s polling loop behind; retries plus the recovery worker piled up dozens of them per stuck ramp (~336 log lines/2min observed in production) until the SquidRouter status API rate-limited us with 429s. - Detect a failed confirmation poll (status "called" + confirm_failed from axelarscan) and auto-recover: fetch a signed ConfirmGatewayTx from Axelar's public recovery signing service and broadcast it to the Axelar RPC, which restarts the validator poll. Uses only the public tx hash - no keys, no funds. Attempts are rate-limited via a cooldown timestamp persisted in ramp state. The byte handling is done manually because axelarjs-sdk's manualRelayToDestChain mangles the relayer's numeric-keyed byte response and broadcasts an empty tx ("must contain at least one message"). --- .../squid-router-pay-phase-handler.test.ts | 236 ++++++++++++++++++ .../squid-router-pay-phase-handler.ts | 70 +++++- .../api/services/phases/meta-state-types.ts | 3 + .../05-integrations/squid-router.md | 3 +- .../src/services/squidrouter/axelar.test.ts | 82 ++++++ .../shared/src/services/squidrouter/axelar.ts | 73 +++++- 6 files changed, 456 insertions(+), 11 deletions(-) create mode 100644 apps/api/src/api/services/phases/handlers/squid-router-pay-phase-handler.test.ts create mode 100644 packages/shared/src/services/squidrouter/axelar.test.ts diff --git a/apps/api/src/api/services/phases/handlers/squid-router-pay-phase-handler.test.ts b/apps/api/src/api/services/phases/handlers/squid-router-pay-phase-handler.test.ts new file mode 100644 index 000000000..980202be5 --- /dev/null +++ b/apps/api/src/api/services/phases/handlers/squid-router-pay-phase-handler.test.ts @@ -0,0 +1,236 @@ +// eslint-disable-next-line import/no-unresolved +import { afterAll, beforeEach, describe, expect, it, mock } from "bun:test"; +// Captured before mock.module so afterAll can restore the real package — +// bun module mocks are process-wide and would poison later test files. +import * as sharedNamespace from "@vortexfi/shared"; +import * as rampServiceNamespace from "../../ramp/ramp.service"; +import * as evmFundingNamespace from "../evm-funding"; + +// Value copies taken before mock.module runs — the namespaces themselves are +// live bindings that would reflect the mocks once installed. +const sharedReal = { ...sharedNamespace }; +const rampServiceReal = { ...rampServiceNamespace }; +const evmFundingReal = { ...evmFundingNamespace }; + +const Networks = { + AssetHub: "assethub", + Base: "base", + Moonbeam: "moonbeam", + Polygon: "polygon" +} as const; + +const FiatToken = { + BRL: "BRL", + EURC: "EUR" +} as const; + +const RampDirection = { + BUY: "BUY", + SELL: "SELL" +} as const; + +const SWAP_HASH = "0x31365ff4337000801303097a0494fd97ecc1661ea84fedee801f01825b236f49"; +const EVM_EPHEMERAL_ADDRESS = "0x1111111111111111111111111111111111111111"; +const FUNDER_ADDRESS = "0x2222222222222222222222222222222222222222"; + +// Queue of axelarscan statuses returned per polling iteration; refilled per test. +let axelarStatusQueue: unknown[] = []; +const getStatusAxelarScan = mock(async () => { + if (axelarStatusQueue.length > 1) { + return axelarStatusQueue.shift(); + } + return axelarStatusQueue[0]; +}); +const getStatus = mock(async () => ({ + id: "", + isGMPTransaction: true, + routeStatus: [], + squidTransactionStatus: "", + status: "ongoing" +})); +const recoverAxelarStuckConfirm = mock(async () => "AXELAR_TX_HASH"); +// Never settles on its own; the real implementation rejects on abort, but these +// tests always resolve via the bridge path of Promise.any. +const checkEvmBalanceForToken = mock(() => new Promise(() => undefined)); + +mock.module("@vortexfi/shared", () => ({ + ...sharedReal, + checkEvmBalanceForToken, + EvmClientManager: { + getInstance: () => ({ + getClient: () => ({}), + getWalletClient: () => ({ account: { address: FUNDER_ADDRESS } }) + }) + }, + FiatToken, + getNetworkId: (network: string) => { + if (network === Networks.Base) return 8453; + if (network === Networks.Polygon) return 137; + if (network === Networks.Moonbeam) return 1284; + return undefined; + }, + getOnChainTokenDetails: () => ({ + decimals: 6, + erc20AddressSourceChain: "0x3333333333333333333333333333333333333333", + isNative: false + }), + getStatus, + getStatusAxelarScan, + isAlfredpayToken: () => false, + Networks, + RampDirection, + recoverAxelarStuckConfirm +})); + +mock.module("../evm-funding", () => ({ + getEvmFundingAccount: () => ({ address: FUNDER_ADDRESS }) +})); + +mock.module("../../ramp/ramp.service", () => ({ + default: { + appendErrorLog: mock(async () => undefined) + } +})); + +const { default: QuoteTicket } = await import("../../../../models/quoteTicket.model"); +const { SquidRouterPayPhaseHandler } = await import("./squid-router-pay-phase-handler"); + +const realQuoteTicketFindByPk = QuoteTicket.findByPk; + +afterAll(() => { + mock.module("@vortexfi/shared", () => ({ ...sharedReal })); + mock.module("../evm-funding", () => ({ ...evmFundingReal })); + mock.module("../../ramp/ramp.service", () => ({ ...rampServiceReal })); + QuoteTicket.findByPk = realQuoteTicketFindByPk; +}); + +let quote: { + inputCurrency: string; + outputCurrency: string; + to: string; +}; + +QuoteTicket.findByPk = mock(async () => quote as any) as typeof QuoteTicket.findByPk; + +function makeState(stateOverrides: Record = {}) { + const state = { + currentPhase: "squidRouterPay", + errorLogs: [], + get() { + const { get: _get, update: _update, ...data } = this; + return data; + }, + id: "ramp-1", + phaseHistory: [], + quoteId: "quote-1", + state: { + evmEphemeralAddress: EVM_EPHEMERAL_ADDRESS, + squidRouterPayTxHash: "0xpay", + squidRouterSwapHash: SWAP_HASH, + ...stateOverrides + }, + to: Networks.Base, + type: RampDirection.BUY, + async update(updateData: Record) { + Object.assign(this, updateData); + return this; + } + }; + return state as any; +} + +function makeHandler() { + const handler = new SquidRouterPayPhaseHandler(); + // Shrink the real 60s/10s waits so the polling loop runs in test time. + (handler as any).initialDelayMs = 10; + (handler as any).pollIntervalMs = 10; + return handler; +} + +const STUCK_CONFIRM_STATUS = { + call: { chain: "base" }, + confirm_failed: true, + id: `${SWAP_HASH}_55_172`, + is_insufficient_fee: false, + status: "called" +}; + +const EXECUTED_STATUS = { + id: `${SWAP_HASH}_55_172`, + is_insufficient_fee: false, + status: "executed" +}; + +describe("SquidRouterPayPhaseHandler", () => { + beforeEach(() => { + axelarStatusQueue = []; + getStatus.mockClear(); + getStatusAxelarScan.mockClear(); + recoverAxelarStuckConfirm.mockClear(); + checkEvmBalanceForToken.mockClear(); + quote = { + inputCurrency: FiatToken.BRL, + outputCurrency: "USDC", + to: Networks.Base + }; + }); + + it("recovers a stuck confirm and records the attempt timestamp", async () => { + axelarStatusQueue = [STUCK_CONFIRM_STATUS, EXECUTED_STATUS]; + + const state = makeState(); + const updatedState = await makeHandler().execute(state); + + expect(recoverAxelarStuckConfirm).toHaveBeenCalledTimes(1); + expect(recoverAxelarStuckConfirm).toHaveBeenCalledWith(SWAP_HASH, "base"); + expect(state.state.axelarConfirmRecoveryAt).toBeString(); + expect(updatedState.currentPhase).toBe("finalSettlementSubsidy"); + }); + + it("respects the cooldown and does not re-broadcast a recent recovery attempt", async () => { + axelarStatusQueue = [STUCK_CONFIRM_STATUS, STUCK_CONFIRM_STATUS, EXECUTED_STATUS]; + + const state = makeState({ axelarConfirmRecoveryAt: new Date().toISOString() }); + await makeHandler().execute(state); + + expect(recoverAxelarStuckConfirm).not.toHaveBeenCalled(); + }); + + it("does not attempt recovery while the confirm poll has not failed", async () => { + axelarStatusQueue = [ + { ...STUCK_CONFIRM_STATUS, confirm_failed: false }, + EXECUTED_STATUS + ]; + + const state = makeState(); + await makeHandler().execute(state); + + expect(recoverAxelarStuckConfirm).not.toHaveBeenCalled(); + }); + + it("stops polling when the processor aborts the execution", async () => { + // Regression test for the retry storm: abandoned executions must unwind on abort + // instead of polling the status APIs forever. + axelarStatusQueue = [STUCK_CONFIRM_STATUS]; + quote = { + inputCurrency: FiatToken.EURC, + outputCurrency: "USDC", + to: Networks.AssetHub + }; + + const abortController = new AbortController(); + const state = makeState({ axelarConfirmRecoveryAt: new Date().toISOString() }); + + const execution = makeHandler().execute(state, abortController.signal); + // Let the loop run a few iterations before aborting. + await new Promise(resolve => setTimeout(resolve, 100)); + abortController.abort(new Error("Phase execution timed out")); + + await expect(execution).rejects.toThrow(); + expect(getStatus.mock.calls.length).toBeGreaterThan(0); + + const callsAtAbort = getStatus.mock.calls.length; + await new Promise(resolve => setTimeout(resolve, 150)); + expect(getStatus.mock.calls.length).toBe(callsAtAbort); + }); +}); diff --git a/apps/api/src/api/services/phases/handlers/squid-router-pay-phase-handler.ts b/apps/api/src/api/services/phases/handlers/squid-router-pay-phase-handler.ts index 3973a004c..87213e120 100644 --- a/apps/api/src/api/services/phases/handlers/squid-router-pay-phase-handler.ts +++ b/apps/api/src/api/services/phases/handlers/squid-router-pay-phase-handler.ts @@ -17,7 +17,9 @@ import { OnChainToken, RampDirection, RampPhase, - SquidRouterPayResponse + recoverAxelarStuckConfirm, + SquidRouterPayResponse, + sleep } from "@vortexfi/shared"; import Big from "big.js"; import { createWalletClient, encodeFunctionData, Hash, PublicClient } from "viem"; @@ -41,6 +43,9 @@ const BALANCE_POLLING_TIME_MS = 10000; // of otherwise successful bridge operations. const EVM_BALANCE_CHECK_TIMEOUT_MS = 15 * 60 * 1000; // 15 minutes const DEFAULT_SQUIDROUTER_GAS_ESTIMATE = "1600000"; // Estimate used to calculate part of the gas fee for SquidRouter transactions. +// Minimum time between Axelar stuck-confirm recovery broadcasts for the same ramp. A new +// validator poll needs a few minutes to complete, so re-broadcasting sooner is pure noise. +const AXELAR_CONFIRM_RECOVERY_COOLDOWN_MS = 10 * 60 * 1000; /** * Handler for the squidRouter pay phase. Checks the status of the Axelar bridge and pays on native GLMR fee. */ @@ -51,6 +56,9 @@ export class SquidRouterPayPhaseHandler extends BasePhaseHandler { private moonbeamWalletClient: ReturnType; private polygonWalletClient: ReturnType; private baseWalletClient: ReturnType; + // Instance fields (not module constants) so tests can shrink the waits. + private initialDelayMs = SQUIDROUTER_INITIAL_DELAY_MS; + private pollIntervalMs = AXELAR_POLLING_INTERVAL_MS; constructor() { super(); @@ -77,7 +85,7 @@ export class SquidRouterPayPhaseHandler extends BasePhaseHandler { * @param state The current ramp state * @returns The updated ramp state */ - protected async executePhase(state: RampState): Promise { + protected async executePhase(state: RampState, signal?: AbortSignal): Promise { const quote = await QuoteTicket.findByPk(state.quoteId); if (!quote) { throw new Error("Quote not found for the given state"); @@ -98,7 +106,7 @@ export class SquidRouterPayPhaseHandler extends BasePhaseHandler { } // Enter check status loop - await this.checkStatus(state, bridgeCallHash, quote); + await this.checkStatus(state, bridgeCallHash, quote, signal); if (state.to === Networks.AssetHub) { return this.transitionToNextPhase(state, "moonbeamToPendulum"); @@ -117,13 +125,13 @@ export class SquidRouterPayPhaseHandler extends BasePhaseHandler { * If the bridge reports success, we consider it a success. * Only if both fail (timeout) we throw. */ - private async checkStatus(state: RampState, swapHash: string, quote: QuoteTicket): Promise { + private async checkStatus(state: RampState, swapHash: string, quote: QuoteTicket, signal?: AbortSignal): Promise { // If the destination is not an EVM network, skip the EVM balance optimization and rely on bridge status only. if (quote.to === Networks.AssetHub) { logger.info("SquidRouterPayPhaseHandler: Destination network is non-EVM; skipping EVM balance check optimization.", { toNetwork: quote.to }); - await this.checkBridgeStatus(state, swapHash, quote); + await this.checkBridgeStatus(state, swapHash, quote, signal); return; } @@ -141,6 +149,7 @@ export class SquidRouterPayPhaseHandler extends BasePhaseHandler { chain: toChain, intervalMs: BALANCE_POLLING_TIME_MS, ownerAddress: ephemeralAddress, + signal, timeoutMs: EVM_BALANCE_CHECK_TIMEOUT_MS, tokenDetails: outTokenDetails }); @@ -156,7 +165,7 @@ export class SquidRouterPayPhaseHandler extends BasePhaseHandler { } // Wrap both promises to prevent unhandled rejections after one succeeds - const bridgeCheckPromise = this.checkBridgeStatus(state, swapHash, quote).catch(err => { + const bridgeCheckPromise = this.checkBridgeStatus(state, swapHash, quote, signal).catch(err => { // Re-throw to preserve the error for Promise.any throw err; }); @@ -199,11 +208,14 @@ export class SquidRouterPayPhaseHandler extends BasePhaseHandler { * Gets the status of the Axelar bridge * @param txHash The swap (bridgeCall) transaction hash */ - private async checkBridgeStatus(state: RampState, swapHash: string, quote: QuoteTicket): Promise { + private async checkBridgeStatus(state: RampState, swapHash: string, quote: QuoteTicket, signal?: AbortSignal): Promise { let isExecuted = false; let payTxHash: string | undefined = state.state.squidRouterPayTxHash; - await new Promise(resolve => setTimeout(resolve, SQUIDROUTER_INITIAL_DELAY_MS)); + // The signal-aware sleeps make abandoned executions unwind when the processor + // times out this phase; without them every timed-out execution left an immortal + // polling loop behind, and they piled up against the SquidRouter rate limit. + await sleep(this.initialDelayMs, signal); while (!isExecuted) { try { @@ -256,6 +268,8 @@ export class SquidRouterPayPhaseHandler extends BasePhaseHandler { await state.update({ state: { ...state.state, squidRouterPayTxHash: payTxHash } }); + } else if (axelarScanStatus.status === "called" && axelarScanStatus.confirm_failed) { + await this.maybeRecoverStuckConfirm(state, swapHash, axelarScanStatus.call?.chain); } } else { logger.info("SquidRouterPayPhaseHandler: Same-chain transaction detected. Skipping Axelar check."); @@ -266,7 +280,45 @@ export class SquidRouterPayPhaseHandler extends BasePhaseHandler { ); } - await new Promise(resolve => setTimeout(resolve, AXELAR_POLLING_INTERVAL_MS)); + await sleep(this.pollIntervalMs, signal); + } + } + + /** + * Axelar's relayer does not retry a failed validator confirmation poll, so a transfer + * whose poll failed stays in status "called" forever. Ask Axelar's recovery signing + * service for a new ConfirmGatewayTx and broadcast it, which restarts the poll. + * Attempts are rate-limited via a timestamp persisted in the ramp state, and failures + * are swallowed so the status loop keeps polling and retries after the cooldown. + */ + private async maybeRecoverStuckConfirm(state: RampState, swapHash: string, sourceChain: string | undefined): Promise { + const lastAttempt = state.state.axelarConfirmRecoveryAt ? new Date(state.state.axelarConfirmRecoveryAt).getTime() : 0; + if (Date.now() - lastAttempt < AXELAR_CONFIRM_RECOVERY_COOLDOWN_MS) { + return; + } + + if (!sourceChain) { + logger.warn( + `SquidRouterPayPhaseHandler: Confirm poll failed for ${swapHash} but Axelar status has no source chain; cannot attempt recovery.` + ); + return; + } + + // Persist the attempt timestamp before broadcasting so a failing relayer is not + // hammered on every 10s poll iteration. + await state.update({ + state: { ...state.state, axelarConfirmRecoveryAt: new Date().toISOString() } + }); + + try { + const axelarTxHash = await recoverAxelarStuckConfirm(swapHash, sourceChain); + logger.info( + `SquidRouterPayPhaseHandler: Confirm poll failed for ${swapHash}; broadcast recovery ConfirmGatewayTx ${axelarTxHash} on Axelar.` + ); + } catch (error) { + logger.warn( + `SquidRouterPayPhaseHandler: Axelar stuck-confirm recovery attempt failed for ${swapHash}: ${error instanceof Error ? error.message : String(error)}` + ); } } diff --git a/apps/api/src/api/services/phases/meta-state-types.ts b/apps/api/src/api/services/phases/meta-state-types.ts index a1b81f1e5..8b5d2fa0f 100644 --- a/apps/api/src/api/services/phases/meta-state-types.ts +++ b/apps/api/src/api/services/phases/meta-state-types.ts @@ -33,6 +33,9 @@ export interface StateMetadata { squidRouterApproveHash: string; squidRouterSwapHash: string; squidRouterPayTxHash: string; + // Timestamp of the last Axelar stuck-confirm recovery attempt, persisted so + // retried phase executions respect the cooldown instead of re-broadcasting. + axelarConfirmRecoveryAt?: string; unhandledPaymentAlertSent: boolean; depositQrCode: string | undefined; // Set to true once update-time validation gate passes (all presigned txs valid + complete, diff --git a/docs/security-spec/05-integrations/squid-router.md b/docs/security-spec/05-integrations/squid-router.md index af31a5841..06e83f11b 100644 --- a/docs/security-spec/05-integrations/squid-router.md +++ b/docs/security-spec/05-integrations/squid-router.md @@ -18,7 +18,7 @@ It handles cross-chain swap execution, Axelar bridge status monitoring, and gas **Chains involved:** Base, Polygon, Moonbeam, Ethereum, Arbitrum, BSC, Avalanche, etc. (any EVM destination supported by Squid) **Phase handlers:** - `squid-router-phase-handler.ts` — Submits presigned approve + swap transactions on the source EVM chain. -- `squid-router-pay-phase-handler.ts` — Monitors Axelar bridge status, funds Axelar gas, waits for cross-chain settlement (with finite arrival timeout). +- `squid-router-pay-phase-handler.ts` — Monitors Axelar bridge status, funds Axelar gas, waits for cross-chain settlement (with finite arrival timeout). Honors the phase processor's `AbortSignal` so timed-out executions stop polling instead of leaking loops against the Squid rate limit. When axelarscan reports a failed validator confirmation poll (`status: "called"` + `confirm_failed` — Axelar's relayer never retries these), it auto-recovers by fetching a signed `ConfirmGatewayTx` from Axelar's public recovery signing service and broadcasting it to the Axelar RPC (`recoverAxelarStuckConfirm` in shared). This uses only the public tx hash — no Vortex keys sign anything and no funds move; attempts are rate-limited by a cooldown timestamp persisted in ramp state (`axelarConfirmRecoveryAt`). - `squidrouter-permit-execution-handler.ts` — Calls `TokenRelayer.execute()` with EIP-2612 permit + payload for off-ramp permit flows. Also handles the no-permit fallback path where the user's wallet submits the substituting transactions directly. ### On-ramp flow (BRL onramp post-Nabla, e.g. Base USDC → user's Polygon ERC-20) @@ -70,6 +70,7 @@ When the BRL on-ramp's destination is **Base + USDC**, the Nabla swap output is | Threat | Mitigation | |---|---| | **Bridge funds stuck in transit** | Dual monitoring (Squid + Axelar scan). 15-minute arrival timeout. Phase retries on failure. Gas proactively funded via `addNativeGas`. | +| **Axelar validator confirm poll fails (transfer stuck at "called")** | Auto-recovery: broadcast a fresh `ConfirmGatewayTx` obtained from Axelar's recovery signing service (public tx hash only, no Vortex keys). Cooldown of 10 minutes between attempts, persisted in ramp state. Recovery failures are swallowed; the status loop keeps polling and retries after the cooldown. | | **Gas overpayment to Axelar** | `calculateGasFeeInUnits()` uses Axelar's reported base fee + estimated gas × source gas price × multiplier. Result verified non-negative. | | **Double-spend of approve/swap** | Approve hash persisted immediately; on re-entry handler skips to swap if hash exists. EVM nonce prevents on-chain double-spend in any case. | | **Permit replay** | Each permit has a nonce + deadline; TokenRelayer validates on-chain. | diff --git a/packages/shared/src/services/squidrouter/axelar.test.ts b/packages/shared/src/services/squidrouter/axelar.test.ts new file mode 100644 index 000000000..d5da12465 --- /dev/null +++ b/packages/shared/src/services/squidrouter/axelar.test.ts @@ -0,0 +1,82 @@ +import { afterAll, afterEach, describe, expect, it, mock } from "bun:test"; +import { recoverAxelarStuckConfirm } from "./axelar"; + +const TX_HASH = "0x31365ff4337000801303097a0494fd97ecc1661ea84fedee801f01825b236f49"; +const SIGNED_TX_BYTES = [10, 137, 1, 42, 0, 255]; + +const realFetch = globalThis.fetch; + +afterEach(() => { + globalThis.fetch = realFetch; +}); + +afterAll(() => { + globalThis.fetch = realFetch; +}); + +function jsonResponse(body: unknown, ok = true, status = 200): Response { + return { + json: async () => body, + ok, + status + } as Response; +} + +describe("recoverAxelarStuckConfirm", () => { + it("decodes the relayer's numeric-keyed byte response and broadcasts it", async () => { + // The signing relayer serializes the tx bytes as {"0": 10, "1": 137, ...}. The + // official SDK mishandles exactly this shape and broadcasts an empty tx, so the + // decode is the load-bearing part of this function. + const numericKeyed: Record = {}; + SIGNED_TX_BYTES.forEach((byte, index) => { + numericKeyed[String(index)] = byte; + }); + + const fetchCalls: { url: string; body: unknown }[] = []; + globalThis.fetch = mock(async (url: string | URL | Request, init?: RequestInit) => { + fetchCalls.push({ body: JSON.parse(init?.body as string), url: String(url) }); + if (String(url).includes("confirm_gateway_tx")) { + return jsonResponse({ data: numericKeyed }); + } + return jsonResponse({ id: 1, jsonrpc: "2.0", result: { code: 0, hash: "ABC123" } }); + }) as typeof fetch; + + const hash = await recoverAxelarStuckConfirm(TX_HASH, "base"); + + expect(hash).toBe("ABC123"); + expect(fetchCalls).toHaveLength(2); + expect(fetchCalls[0].url).toContain("axelar-signing-relayer-mainnet.axelar.dev/confirm_gateway_tx"); + expect(fetchCalls[0].body).toEqual({ chain: "base", module: "evm", txHash: TX_HASH }); + + const expectedBase64 = btoa(String.fromCharCode(...SIGNED_TX_BYTES)); + expect(fetchCalls[1].body).toMatchObject({ method: "broadcast_tx_sync", params: { tx: expectedBase64 } }); + }); + + it("accepts a plain byte array from the relayer", async () => { + globalThis.fetch = mock(async (url: string | URL | Request) => { + if (String(url).includes("confirm_gateway_tx")) { + return jsonResponse({ data: SIGNED_TX_BYTES }); + } + return jsonResponse({ id: 1, jsonrpc: "2.0", result: { code: 0, hash: "DEF456" } }); + }) as typeof fetch; + + await expect(recoverAxelarStuckConfirm(TX_HASH, "base")).resolves.toBe("DEF456"); + }); + + it("throws when the relayer returns an empty transaction instead of broadcasting it", async () => { + globalThis.fetch = mock(async () => jsonResponse({ data: {} })) as typeof fetch; + + await expect(recoverAxelarStuckConfirm(TX_HASH, "base")).rejects.toThrow("empty transaction"); + }); + + it("throws when the Axelar broadcast is rejected", async () => { + globalThis.fetch = mock(async (url: string | URL | Request) => { + if (String(url).includes("confirm_gateway_tx")) { + return jsonResponse({ data: SIGNED_TX_BYTES }); + } + return jsonResponse({ id: 1, jsonrpc: "2.0", result: { code: 18, log: "must contain at least one message" } }); + }) as typeof fetch; + + await expect(recoverAxelarStuckConfirm(TX_HASH, "base")).rejects.toThrow("code 18"); + }); +}); diff --git a/packages/shared/src/services/squidrouter/axelar.ts b/packages/shared/src/services/squidrouter/axelar.ts index a2fe33c37..e46ee1917 100644 --- a/packages/shared/src/services/squidrouter/axelar.ts +++ b/packages/shared/src/services/squidrouter/axelar.ts @@ -21,12 +21,83 @@ export interface AxelarScanStatusFees { execute_gas_multiplier: number; } -interface AxelarScanStatusResponse { +export interface AxelarScanStatusResponse { is_insufficient_fee: boolean; status: string; // executed or express_executed (for complete). fees: AxelarScanStatusFees; id: string; // the id of the swap. + // Set by axelarscan when the validator poll confirming the source event failed. + // Axelar's own relayer does not retry a failed poll, so the transfer stays in + // status "called" until a new ConfirmGatewayTx is broadcast. + confirm_failed?: boolean; + call?: { + chain: string; // source chain in Axelar naming, e.g. "base" + }; +} +const AXELAR_SIGNING_RELAYER_URL = "https://axelar-signing-relayer-mainnet.axelar.dev"; +const AXELAR_RPC_URL = "https://mainnet.rpc.axelar.dev/chain/axelar"; + +/** + * Recovers a GMP transfer stuck at the confirmation step (status "called" with + * confirm_failed) by asking Axelar's recovery signing service for a signed + * ConfirmGatewayTx and broadcasting it to the Axelar network. This restarts the + * validator poll; once it passes, approval and execution proceed automatically. + * + * Uses only the public tx hash — no wallet or keys are involved. The official + * axelarjs-sdk `manualRelayToDestChain` performs the same steps but mangles the + * relayer's byte response (numeric-keyed JSON) and broadcasts an empty tx, so we + * do the byte handling and broadcast ourselves. + * + * @param txHash The source-chain transaction hash of the stuck GMP call + * @param sourceChain The source chain in Axelar naming (e.g. "base") + * @returns The Axelar transaction hash of the broadcast ConfirmGatewayTx + */ +export async function recoverAxelarStuckConfirm(txHash: string, sourceChain: string): Promise { + const relayerResponse = await fetch(`${AXELAR_SIGNING_RELAYER_URL}/confirm_gateway_tx`, { + body: JSON.stringify({ chain: sourceChain, module: "evm", txHash }), + headers: { "Content-Type": "application/json" }, + method: "POST" + }); + if (!relayerResponse.ok) { + throw new Error(`Axelar signing relayer returned HTTP ${relayerResponse.status}`); + } + + // The relayer returns the signed tx bytes as JSON: either a Buffer-style + // {data: [..]} array or a numeric-keyed object {data: {"0": 10, "1": 137, ...}}. + const relayerJson = (await relayerResponse.json()) as { data?: unknown }; + const rawBytes = relayerJson.data ?? relayerJson; + const byteValues: number[] = Array.isArray(rawBytes) + ? rawBytes + : Object.keys(rawBytes as Record) + .sort((a, b) => Number(a) - Number(b)) + .map(key => (rawBytes as Record)[key]); + if (byteValues.length === 0) { + throw new Error("Axelar signing relayer returned an empty transaction"); + } + + let binary = ""; + for (const byte of byteValues) { + binary += String.fromCharCode(byte); + } + const txBase64 = btoa(binary); + + const rpcResponse = await fetch(AXELAR_RPC_URL, { + body: JSON.stringify({ id: 1, jsonrpc: "2.0", method: "broadcast_tx_sync", params: { tx: txBase64 } }), + headers: { "Content-Type": "application/json" }, + method: "POST" + }); + if (!rpcResponse.ok) { + throw new Error(`Axelar RPC returned HTTP ${rpcResponse.status}`); + } + + const rpcJson = (await rpcResponse.json()) as { result?: { code?: number; hash?: string; log?: string } }; + if (!rpcJson.result || rpcJson.result.code !== 0) { + throw new Error(`Axelar broadcast failed with code ${rpcJson.result?.code}: ${rpcJson.result?.log ?? "unknown error"}`); + } + + return rpcJson.result.hash ?? ""; } + export async function getStatusAxelarScan(swapHash: string): Promise { try { // POST call, https://api.axelarscan.io/gmp/searchGMP From ef6e15cfdf5cee4410ce1b6d410804dda23904cc Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Sun, 19 Jul 2026 16:28:56 +0200 Subject: [PATCH 2/3] fix(api): harden Axelar stuck-confirm recovery per review feedback Addresses Copilot review on #1276: - recoverAxelarStuckConfirm: validate the relayer response shape (reject null/non-object data, ignore non-numeric keys, require integer bytes in 0-255) instead of silently coercing garbage into a corrupted broadcast. - Treat a successful broadcast without a returned tx hash as an error rather than returning an empty string. - Thread the phase AbortSignal into the recovery fetches so aborted executions stop promptly instead of finishing network I/O after timeout. - Normalize an unparseable persisted axelarConfirmRecoveryAt to "never attempted" so the cooldown comparison stays well-defined. --- .../squid-router-pay-phase-handler.test.ts | 2 +- .../squid-router-pay-phase-handler.ts | 16 ++++++-- .../src/services/squidrouter/axelar.test.ts | 41 +++++++++++++++++++ .../shared/src/services/squidrouter/axelar.ts | 38 ++++++++++++----- 4 files changed, 81 insertions(+), 16 deletions(-) diff --git a/apps/api/src/api/services/phases/handlers/squid-router-pay-phase-handler.test.ts b/apps/api/src/api/services/phases/handlers/squid-router-pay-phase-handler.test.ts index 980202be5..9c2389b33 100644 --- a/apps/api/src/api/services/phases/handlers/squid-router-pay-phase-handler.test.ts +++ b/apps/api/src/api/services/phases/handlers/squid-router-pay-phase-handler.test.ts @@ -182,7 +182,7 @@ describe("SquidRouterPayPhaseHandler", () => { const updatedState = await makeHandler().execute(state); expect(recoverAxelarStuckConfirm).toHaveBeenCalledTimes(1); - expect(recoverAxelarStuckConfirm).toHaveBeenCalledWith(SWAP_HASH, "base"); + expect(recoverAxelarStuckConfirm).toHaveBeenCalledWith(SWAP_HASH, "base", undefined); expect(state.state.axelarConfirmRecoveryAt).toBeString(); expect(updatedState.currentPhase).toBe("finalSettlementSubsidy"); }); diff --git a/apps/api/src/api/services/phases/handlers/squid-router-pay-phase-handler.ts b/apps/api/src/api/services/phases/handlers/squid-router-pay-phase-handler.ts index 87213e120..9396f820d 100644 --- a/apps/api/src/api/services/phases/handlers/squid-router-pay-phase-handler.ts +++ b/apps/api/src/api/services/phases/handlers/squid-router-pay-phase-handler.ts @@ -269,7 +269,7 @@ export class SquidRouterPayPhaseHandler extends BasePhaseHandler { state: { ...state.state, squidRouterPayTxHash: payTxHash } }); } else if (axelarScanStatus.status === "called" && axelarScanStatus.confirm_failed) { - await this.maybeRecoverStuckConfirm(state, swapHash, axelarScanStatus.call?.chain); + await this.maybeRecoverStuckConfirm(state, swapHash, axelarScanStatus.call?.chain, signal); } } else { logger.info("SquidRouterPayPhaseHandler: Same-chain transaction detected. Skipping Axelar check."); @@ -291,8 +291,16 @@ export class SquidRouterPayPhaseHandler extends BasePhaseHandler { * Attempts are rate-limited via a timestamp persisted in the ramp state, and failures * are swallowed so the status loop keeps polling and retries after the cooldown. */ - private async maybeRecoverStuckConfirm(state: RampState, swapHash: string, sourceChain: string | undefined): Promise { - const lastAttempt = state.state.axelarConfirmRecoveryAt ? new Date(state.state.axelarConfirmRecoveryAt).getTime() : 0; + private async maybeRecoverStuckConfirm( + state: RampState, + swapHash: string, + sourceChain: string | undefined, + signal?: AbortSignal + ): Promise { + // An unparseable persisted timestamp yields NaN; treat it as "never attempted" so + // the comparison below stays well-defined (NaN comparisons are always false). + const parsedLastAttempt = state.state.axelarConfirmRecoveryAt ? new Date(state.state.axelarConfirmRecoveryAt).getTime() : 0; + const lastAttempt = Number.isFinite(parsedLastAttempt) ? parsedLastAttempt : 0; if (Date.now() - lastAttempt < AXELAR_CONFIRM_RECOVERY_COOLDOWN_MS) { return; } @@ -311,7 +319,7 @@ export class SquidRouterPayPhaseHandler extends BasePhaseHandler { }); try { - const axelarTxHash = await recoverAxelarStuckConfirm(swapHash, sourceChain); + const axelarTxHash = await recoverAxelarStuckConfirm(swapHash, sourceChain, signal); logger.info( `SquidRouterPayPhaseHandler: Confirm poll failed for ${swapHash}; broadcast recovery ConfirmGatewayTx ${axelarTxHash} on Axelar.` ); diff --git a/packages/shared/src/services/squidrouter/axelar.test.ts b/packages/shared/src/services/squidrouter/axelar.test.ts index d5da12465..295f15344 100644 --- a/packages/shared/src/services/squidrouter/axelar.test.ts +++ b/packages/shared/src/services/squidrouter/axelar.test.ts @@ -69,6 +69,47 @@ describe("recoverAxelarStuckConfirm", () => { await expect(recoverAxelarStuckConfirm(TX_HASH, "base")).rejects.toThrow("empty transaction"); }); + it("ignores non-numeric keys in the relayer response", async () => { + const fetchCalls: { url: string; body: unknown }[] = []; + globalThis.fetch = mock(async (url: string | URL | Request, init?: RequestInit) => { + fetchCalls.push({ body: JSON.parse(init?.body as string), url: String(url) }); + if (String(url).includes("confirm_gateway_tx")) { + return jsonResponse({ data: { "0": 10, "1": 137, type: "Buffer" } }); + } + return jsonResponse({ id: 1, jsonrpc: "2.0", result: { code: 0, hash: "GHI789" } }); + }) as typeof fetch; + + await expect(recoverAxelarStuckConfirm(TX_HASH, "base")).resolves.toBe("GHI789"); + expect(fetchCalls[1].body).toMatchObject({ params: { tx: btoa(String.fromCharCode(10, 137)) } }); + }); + + it("throws instead of broadcasting corrupted bytes when the relayer response has an unexpected shape", async () => { + for (const data of [null, undefined, "not-bytes", { "0": 300 }, { "0": "10" }, [10, 1.5]]) { + globalThis.fetch = mock(async () => jsonResponse({ data })) as typeof fetch; + await expect(recoverAxelarStuckConfirm(TX_HASH, "base")).rejects.toThrow(/unexpected response shape|invalid transaction bytes/); + } + }); + + it("throws when the broadcast succeeds but the RPC response has no transaction hash", async () => { + globalThis.fetch = mock(async (url: string | URL | Request) => { + if (String(url).includes("confirm_gateway_tx")) { + return jsonResponse({ data: SIGNED_TX_BYTES }); + } + return jsonResponse({ id: 1, jsonrpc: "2.0", result: { code: 0 } }); + }) as typeof fetch; + + await expect(recoverAxelarStuckConfirm(TX_HASH, "base")).rejects.toThrow("no transaction hash"); + }); + + it("aborts the relayer call when the signal fires", async () => { + globalThis.fetch = realFetch; + const abortController = new AbortController(); + abortController.abort(new Error("Phase execution timed out")); + + // With an already-aborted signal, fetch must reject without any network I/O. + await expect(recoverAxelarStuckConfirm(TX_HASH, "base", abortController.signal)).rejects.toThrow(); + }); + it("throws when the Axelar broadcast is rejected", async () => { globalThis.fetch = mock(async (url: string | URL | Request) => { if (String(url).includes("confirm_gateway_tx")) { diff --git a/packages/shared/src/services/squidrouter/axelar.ts b/packages/shared/src/services/squidrouter/axelar.ts index e46ee1917..554d19286 100644 --- a/packages/shared/src/services/squidrouter/axelar.ts +++ b/packages/shared/src/services/squidrouter/axelar.ts @@ -50,13 +50,15 @@ const AXELAR_RPC_URL = "https://mainnet.rpc.axelar.dev/chain/axelar"; * * @param txHash The source-chain transaction hash of the stuck GMP call * @param sourceChain The source chain in Axelar naming (e.g. "base") + * @param signal Aborts the recovery's network calls when the caller gives up * @returns The Axelar transaction hash of the broadcast ConfirmGatewayTx */ -export async function recoverAxelarStuckConfirm(txHash: string, sourceChain: string): Promise { +export async function recoverAxelarStuckConfirm(txHash: string, sourceChain: string, signal?: AbortSignal): Promise { const relayerResponse = await fetch(`${AXELAR_SIGNING_RELAYER_URL}/confirm_gateway_tx`, { body: JSON.stringify({ chain: sourceChain, module: "evm", txHash }), headers: { "Content-Type": "application/json" }, - method: "POST" + method: "POST", + signal }); if (!relayerResponse.ok) { throw new Error(`Axelar signing relayer returned HTTP ${relayerResponse.status}`); @@ -64,19 +66,29 @@ export async function recoverAxelarStuckConfirm(txHash: string, sourceChain: str // The relayer returns the signed tx bytes as JSON: either a Buffer-style // {data: [..]} array or a numeric-keyed object {data: {"0": 10, "1": 137, ...}}. + // Anything else is rejected rather than silently coerced into corrupt bytes. const relayerJson = (await relayerResponse.json()) as { data?: unknown }; - const rawBytes = relayerJson.data ?? relayerJson; - const byteValues: number[] = Array.isArray(rawBytes) - ? rawBytes - : Object.keys(rawBytes as Record) - .sort((a, b) => Number(a) - Number(b)) - .map(key => (rawBytes as Record)[key]); + const rawBytes = relayerJson.data; + let byteValues: unknown[]; + if (Array.isArray(rawBytes)) { + byteValues = rawBytes; + } else if (rawBytes !== null && typeof rawBytes === "object") { + byteValues = Object.keys(rawBytes) + .filter(key => /^\d+$/.test(key)) + .sort((a, b) => Number(a) - Number(b)) + .map(key => (rawBytes as Record)[key]); + } else { + throw new Error("Axelar signing relayer returned an unexpected response shape"); + } if (byteValues.length === 0) { throw new Error("Axelar signing relayer returned an empty transaction"); } + if (!byteValues.every(byte => typeof byte === "number" && Number.isInteger(byte) && byte >= 0 && byte <= 255)) { + throw new Error("Axelar signing relayer returned invalid transaction bytes"); + } let binary = ""; - for (const byte of byteValues) { + for (const byte of byteValues as number[]) { binary += String.fromCharCode(byte); } const txBase64 = btoa(binary); @@ -84,7 +96,8 @@ export async function recoverAxelarStuckConfirm(txHash: string, sourceChain: str const rpcResponse = await fetch(AXELAR_RPC_URL, { body: JSON.stringify({ id: 1, jsonrpc: "2.0", method: "broadcast_tx_sync", params: { tx: txBase64 } }), headers: { "Content-Type": "application/json" }, - method: "POST" + method: "POST", + signal }); if (!rpcResponse.ok) { throw new Error(`Axelar RPC returned HTTP ${rpcResponse.status}`); @@ -94,8 +107,11 @@ export async function recoverAxelarStuckConfirm(txHash: string, sourceChain: str if (!rpcJson.result || rpcJson.result.code !== 0) { throw new Error(`Axelar broadcast failed with code ${rpcJson.result?.code}: ${rpcJson.result?.log ?? "unknown error"}`); } + if (!rpcJson.result.hash) { + throw new Error("Axelar broadcast succeeded but the RPC response contained no transaction hash"); + } - return rpcJson.result.hash ?? ""; + return rpcJson.result.hash; } export async function getStatusAxelarScan(swapHash: string): Promise { From 597c0e397aa81d0abf7a3559272b5ab6a8f7ae1a Mon Sep 17 00:00:00 2001 From: Marcel Ebert Date: Sun, 19 Jul 2026 16:46:26 +0200 Subject: [PATCH 3/3] perf(shared): chunk base64 encoding of the recovery tx bytes Addresses Copilot follow-up on #1276: byte-by-byte string concatenation before btoa is quadratic; encode in 8KiB String.fromCharCode chunks instead (spreading the full array at once could hit the argument-count limit). --- packages/shared/src/services/squidrouter/axelar.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/shared/src/services/squidrouter/axelar.ts b/packages/shared/src/services/squidrouter/axelar.ts index 554d19286..a881a70eb 100644 --- a/packages/shared/src/services/squidrouter/axelar.ts +++ b/packages/shared/src/services/squidrouter/axelar.ts @@ -87,11 +87,14 @@ export async function recoverAxelarStuckConfirm(txHash: string, sourceChain: str throw new Error("Axelar signing relayer returned invalid transaction bytes"); } - let binary = ""; - for (const byte of byteValues as number[]) { - binary += String.fromCharCode(byte); + // Encode in chunks: one String.fromCharCode call per byte is quadratic on large + // arrays, while spreading the whole array at once risks the argument-count limit. + const CHUNK_SIZE = 0x2000; + const binaryChunks: string[] = []; + for (let i = 0; i < byteValues.length; i += CHUNK_SIZE) { + binaryChunks.push(String.fromCharCode(...(byteValues.slice(i, i + CHUNK_SIZE) as number[]))); } - const txBase64 = btoa(binary); + const txBase64 = btoa(binaryChunks.join("")); const rpcResponse = await fetch(AXELAR_RPC_URL, { body: JSON.stringify({ id: 1, jsonrpc: "2.0", method: "broadcast_tx_sync", params: { tx: txBase64 } }),