From 3480380cd113c6dea42e0769f8eb0607427fbb76 Mon Sep 17 00:00:00 2001 From: Vedant Madane <6527493+VedantMadane@users.noreply.github.com> Date: Mon, 24 Aug 2026 09:55:59 +0530 Subject: [PATCH] feat(keeper): add retry logic with exponential backoff for failed charges Distinguish transient RPC/network failures from permanent contract errors, retry with exponential backoff (max 3), and confirm via transaction status lookup that a charge did not already succeed before retrying. Fixes #7 Signed-off-by: Vedant Madane <6527493+VedantMadane@users.noreply.github.com> --- keeper/keeper.js | 131 +++++++++++++++++++++++++++++++++++--- keeper/retry.js | 134 +++++++++++++++++++++++++++++++++++++++ keeper/simulate_retry.js | 84 ++++++++++++++++++++++++ 3 files changed, 341 insertions(+), 8 deletions(-) create mode 100644 keeper/retry.js create mode 100644 keeper/simulate_retry.js diff --git a/keeper/keeper.js b/keeper/keeper.js index 1fa9061..cdc6868 100644 --- a/keeper/keeper.js +++ b/keeper/keeper.js @@ -1,4 +1,4 @@ -const { +const { Contract, SorobanRpc, TransactionBuilder, @@ -7,17 +7,45 @@ const { nativeToScVal, BASE_FEE, } = require("@stellar/stellar-sdk"); +const { withRetry, isRetryableError, isPermanentError } = require("./retry"); -const RPC_URL = "https://soroban-testnet.stellar.org"; -const CONTRACT_ID = "CCIL5WPQB4KGYD5ITC5TKSQEI7V7L4CQM623TNF6GSKNR44H3CI2OTSR"; +const RPC_URL = process.env.RPC_URL || "https://soroban-testnet.stellar.org"; +const CONTRACT_ID = + process.env.CONTRACT_ID || + "CCIL5WPQB4KGYD5ITC5TKSQEI7V7L4CQM623TNF6GSKNR44H3CI2OTSR"; const KEEPER_SECRET = process.env.KEEPER_SECRET_KEY; -const SUBSCRIPTION_IDS = [0, 1, 2]; // v1: hardcoded, replace with real IDs +const SUBSCRIPTION_IDS = (process.env.SUBSCRIPTION_IDS || "0,1,2") + .split(",") + .map((s) => Number(s.trim())) + .filter((n) => Number.isFinite(n)); + +if (!KEEPER_SECRET) { + console.error("KEEPER_SECRET_KEY is required"); + process.exit(1); +} const server = new SorobanRpc.Server(RPC_URL); const keeperKeypair = Keypair.fromSecret(KEEPER_SECRET); const contract = new Contract(CONTRACT_ID); -async function chargeSubscription(subId) { +/** + * Look up a submitted transaction. Returns true only when the network has + * confirmed SUCCESS — used to avoid retrying a charge that already settled. + */ +async function transactionSucceeded(hash) { + if (!hash) return false; + try { + const tx = await server.getTransaction(hash); + // sdk status enum: SUCCESS | NOT_FOUND | FAILED + return tx && tx.status === "SUCCESS"; + } catch (err) { + // Lookup failure is not proof of success; treat as "not confirmed". + console.log(`tx lookup ${hash}: ${err.message}`); + return false; + } +} + +async function sendChargeOnce(subId) { const account = await server.getAccount(keeperKeypair.publicKey()); const tx = new TransactionBuilder(account, { fee: BASE_FEE, @@ -30,11 +58,85 @@ async function chargeSubscription(subId) { const prepared = await server.prepareTransaction(tx); prepared.sign(keeperKeypair); + const result = await server.sendTransaction(prepared); + const hash = result.hash || result.id; + + if (result.status === "ERROR" || result.status === "FAILED") { + const err = new Error( + `sendTransaction status=${result.status} for subscription ${subId}` + ); + err.result = result; + err.hash = hash; + throw err; + } + + // PENDING / TRY_AGAIN_LATER / DUPLICATE — wait for confirmation when we have a hash. + if (hash) { + // Brief poll: transient inclusion lag should not look like a failed charge. + for (let i = 0; i < 5; i++) { + if (await transactionSucceeded(hash)) { + return { status: "SUCCESS", hash, subId }; + } + await new Promise((r) => setTimeout(r, 1000 * (i + 1))); + } + // Not confirmed yet — if the tx might still land, surface as retryable + // only after we verified it has not succeeded. + const err = new Error( + `charge not confirmed for subscription ${subId} (hash=${hash})` + ); + err.hash = hash; + err.code = "NOT_CONFIRMED"; + throw err; + } + + return { status: result.status, hash, subId }; +} + +async function chargeSubscription(subId) { try { - const result = await server.sendTransaction(prepared); - console.log(`Charged subscription ${subId}: ${result.status}`); + const outcome = await withRetry( + () => sendChargeOnce(subId), + { + maxAttempts: 3, + label: `charge(sub=${subId})`, + // Before a retry, ensure a prior submission did not already succeed. + shouldRetry: async (err, attempt) => { + if (isPermanentError(err)) { + console.log( + `permanent failure for subscription ${subId} (attempt ${attempt}): ${err.message}` + ); + return false; + } + if (err && err.hash) { + const done = await transactionSucceeded(err.hash); + if (done) { + console.log( + `subscription ${subId}: prior tx ${err.hash} already SUCCESS — not retrying` + ); + return false; + } + } + if (!isRetryableError(err)) { + console.log( + `non-retryable failure for subscription ${subId} (attempt ${attempt}): ${err.message}` + ); + return false; + } + console.log( + `retryable failure for subscription ${subId} (attempt ${attempt}): ${err.message}` + ); + return true; + }, + } + ); + console.log( + `Charged subscription ${subId}: ${outcome.status}` + + (outcome.hash ? ` hash=${outcome.hash}` : "") + ); + return outcome; } catch (err) { console.log(`Skipped subscription ${subId}: ${err.message}`); + return null; } } @@ -44,4 +146,17 @@ async function run() { } } -run(); +if (require.main === module) { + run().catch((err) => { + console.error(err); + process.exit(1); + }); +} + +module.exports = { + chargeSubscription, + transactionSucceeded, + sendChargeOnce, + isRetryableError, + isPermanentError, +}; diff --git a/keeper/retry.js b/keeper/retry.js new file mode 100644 index 0000000..0c99715 --- /dev/null +++ b/keeper/retry.js @@ -0,0 +1,134 @@ +/** + * Retry helper with exponential backoff for keeper charge attempts. + * + * Distinguishes transient failures (RPC/network) from permanent contract + * errors (Unauthorized, insufficient allowance, not due, cancelled) so we + * never hammer a subscription that cannot succeed. + */ + +const DEFAULT_MAX_ATTEMPTS = 3; +const DEFAULT_BASE_DELAY_MS = 500; +const DEFAULT_MAX_DELAY_MS = 8000; + +/** Contract / auth failures that will not succeed on retry. */ +const PERMANENT_PATTERNS = [ + /unauthorized/i, + /insufficient.?allowance/i, + /not.?due/i, + /cancelled|canceled/i, + /already.?charged/i, + /invalid.?subscription/i, + /HostError/i, + /UnreachableCodeReached/i, + /InvalidAction/i, +]; + +/** Network / RPC blips worth retrying. */ +const RETRYABLE_PATTERNS = [ + /timeout/i, + /ECONNRESET/i, + /ECONNREFUSED/i, + /ENOTFOUND/i, + /ETIMEDOUT/i, + /socket hang up/i, + /network/i, + /429/, + /502/, + /503/, + /504/, + /TRY_AGAIN/i, + /NOT_CONFIRMED/i, + /connection/i, + /fetch failed/i, + /temporarily unavailable/i, +]; + +function errorText(err) { + if (!err) return ""; + const parts = [err.message, err.code, err.name]; + if (err.response && err.response.data) { + parts.push(JSON.stringify(err.response.data)); + } + if (err.result) { + parts.push(JSON.stringify(err.result)); + } + return parts.filter(Boolean).join(" "); +} + +function isPermanentError(err) { + const text = errorText(err); + return PERMANENT_PATTERNS.some((re) => re.test(text)); +} + +function isRetryableError(err) { + if (isPermanentError(err)) return false; + const text = errorText(err); + if (RETRYABLE_PATTERNS.some((re) => re.test(text))) return true; + // Unknown errors: retry once-class — treat as retryable up to the cap so a + // flaky RPC is not silently dropped. Permanent patterns above still win. + return true; +} + +function delay(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function backoffMs(attempt, baseDelayMs, maxDelayMs) { + // attempt is 1-based for the failure that just happened + const exp = Math.min(maxDelayMs, baseDelayMs * 2 ** (attempt - 1)); + const jitter = Math.floor(Math.random() * Math.min(250, exp * 0.1)); + return exp + jitter; +} + +/** + * @template T + * @param {() => Promise} fn + * @param {object} [opts] + * @param {number} [opts.maxAttempts] + * @param {number} [opts.baseDelayMs] + * @param {number} [opts.maxDelayMs] + * @param {string} [opts.label] + * @param {(err: Error, attempt: number) => boolean | Promise} [opts.shouldRetry] + * @returns {Promise} + */ +async function withRetry(fn, opts = {}) { + const maxAttempts = opts.maxAttempts ?? DEFAULT_MAX_ATTEMPTS; + const baseDelayMs = opts.baseDelayMs ?? DEFAULT_BASE_DELAY_MS; + const maxDelayMs = opts.maxDelayMs ?? DEFAULT_MAX_DELAY_MS; + const label = opts.label || "operation"; + const shouldRetry = opts.shouldRetry || ((err) => isRetryableError(err)); + + let lastErr; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + const result = await fn(); + if (attempt > 1) { + console.log(`${label}: succeeded on attempt ${attempt}/${maxAttempts}`); + } + return result; + } catch (err) { + lastErr = err; + const retry = + attempt < maxAttempts ? await shouldRetry(err, attempt) : false; + console.log( + `${label}: attempt ${attempt}/${maxAttempts} failed: ${err.message}` + + (retry ? " — will retry" : " — giving up") + ); + if (!retry) break; + const wait = backoffMs(attempt, baseDelayMs, maxDelayMs); + console.log(`${label}: backing off ${wait}ms before attempt ${attempt + 1}`); + await delay(wait); + } + } + throw lastErr; +} + +module.exports = { + withRetry, + isRetryableError, + isPermanentError, + backoffMs, + DEFAULT_MAX_ATTEMPTS, + PERMANENT_PATTERNS, + RETRYABLE_PATTERNS, +}; diff --git a/keeper/simulate_retry.js b/keeper/simulate_retry.js new file mode 100644 index 0000000..2186532 --- /dev/null +++ b/keeper/simulate_retry.js @@ -0,0 +1,84 @@ +/** + * Simulated run log: transient failure recovers on retry; permanent is skipped. + * Run: node simulate_retry.js + */ +const { withRetry, isRetryableError, isPermanentError } = require("./retry"); + +async function simulate() { + console.log("=== simulate transient RPC failure then success ==="); + let n = 0; + const ok = await withRetry( + async () => { + n += 1; + if (n < 3) { + const err = new Error("network timeout talking to RPC"); + err.code = "ETIMEDOUT"; + throw err; + } + return { status: "SUCCESS", hash: "SIMULATED_HASH" }; + }, + { maxAttempts: 3, label: "charge(sub=0)", baseDelayMs: 10, maxDelayMs: 20 } + ); + console.log("result:", ok); + + console.log("\n=== simulate permanent insufficient allowance ==="); + try { + await withRetry( + async () => { + const err = new Error("HostError: Error(Contract, #2) insufficient allowance"); + throw err; + }, + { + maxAttempts: 3, + label: "charge(sub=1)", + baseDelayMs: 10, + maxDelayMs: 20, + shouldRetry: async (err, attempt) => { + if (isPermanentError(err)) { + console.log(`permanent (attempt ${attempt}): ${err.message}`); + return false; + } + return isRetryableError(err); + }, + } + ); + } catch (err) { + console.log("gave up:", err.message); + } + + console.log("\n=== simulate prior tx already SUCCESS (no double-charge) ==="); + let sends = 0; + const priorHash = "ALREADY_LANDED"; + try { + await withRetry( + async () => { + sends += 1; + const err = new Error("charge not confirmed"); + err.hash = priorHash; + err.code = "NOT_CONFIRMED"; + throw err; + }, + { + maxAttempts: 3, + label: "charge(sub=2)", + baseDelayMs: 10, + maxDelayMs: 20, + shouldRetry: async (err) => { + // Pretend chain lookup found SUCCESS + if (err.hash === priorHash) { + console.log(`prior tx ${err.hash} already SUCCESS — not retrying`); + return false; + } + return isRetryableError(err); + }, + } + ); + } catch (err) { + console.log("gave up after sends=", sends, "err=", err.message); + } +} + +simulate().catch((e) => { + console.error(e); + process.exit(1); +});