From 06d3cab5823f5f9818d5243c0f37ba4a2a6d1164 Mon Sep 17 00:00:00 2001 From: Robert Leifke Date: Fri, 18 Sep 2026 01:23:45 -0400 Subject: [PATCH] feat(rebalance): alert when the market maker needs cNGN, on share not on idle USDC The failure this watches for is the bid side going dark. The market maker's flow is one-directional -- it sells cNGN for USDC -- so cNGN drains while USDC piles up, and the first visible symptom is an order book with no bids. By then the venue has stopped quoting one side. `pnpm rebalance check [--alert]` reads the subaccount, values both legs through the same HyperFX feed a rebalance would trade at, and says whether to act. The trigger is cNGN's SHARE of inventory value, not an absolute USDC figure. The first version alerted on "idle USDC over $200" and, run against live state, fired on sub 15 holding $310 USDC against $348 of cNGN -- a balanced book where converting would have worsened the imbalance. Caught by running it, not by reading it. A threshold that is wrong the first time it runs is one an operator learns to ignore, so `inventory.test.ts` pins those exact balances as healthy. cNGN under $100 urgent bids about to go dark cNGN under 35% of inventory rebalance convert some USDC USDC within 20% of the $800 halt noted in the message `--alert` refuses to run without ALERT_WEBHOOK_URL rather than logging and exiting 0, and posts the canary's `{text, content}` shape so one webhook serves Slack and Discord. Why this is an alert and not an automated loop: withdrawals pay out only to the subaccount owner -- the action data is `(asset, amount)` with no recipient -- and assertWithdrawalPolicy refuses any withdrawal whose signer is not the owner ("session-key withdrawals are not supported"). USDC leaving sub 15 therefore lands at the market maker's wallet, signed by the market maker's key, and no delegation to the rebalance signer is possible. Automating it properly means the market maker doing the withdrawal itself, which is a change to the Go service. Granting this service kms:Sign on the MM key would also work and should not be done: KMS grants are not partial, so it would confer full market-maker authority and undo the separation this key exists to preserve. Co-authored-by: Claude Opus 5 (1M context) --- services/rebalance/README.md | 36 +++++++++- services/rebalance/src/alert.ts | 17 +++++ services/rebalance/src/check.ts | 54 +++++++++++++++ services/rebalance/src/cli.ts | 5 +- services/rebalance/src/config.ts | 19 ++++++ services/rebalance/src/inventory.test.ts | 65 ++++++++++++++++++ services/rebalance/src/inventory.ts | 87 ++++++++++++++++++++++++ services/rebalance/src/venue.ts | 2 + 8 files changed, 281 insertions(+), 4 deletions(-) create mode 100644 services/rebalance/src/alert.ts create mode 100644 services/rebalance/src/check.ts create mode 100644 services/rebalance/src/inventory.test.ts create mode 100644 services/rebalance/src/inventory.ts diff --git a/services/rebalance/README.md b/services/rebalance/README.md index a3f6e62..fe15bbd 100644 --- a/services/rebalance/README.md +++ b/services/rebalance/README.md @@ -18,6 +18,8 @@ Everything is a dry run unless `--execute` is passed. ```bash export BASE_RPC_URL=... # keyed endpoint, see below +pnpm rebalance check # is a rebalance due? (for a timer) +pnpm rebalance check --alert # ...and post to the ops webhook if so pnpm rebalance quote 20 # what the live feed prices it at pnpm rebalance approve 20 --execute pnpm rebalance swap 20 --execute # place, auction, fill @@ -76,8 +78,36 @@ the balance first (see `infra/aws/secrets.tf`). The signer needs ETH for gas — a whole three-leg cycle costs about $0.02 at current Base prices. +## When to rebalance + +`check` reads the subaccount and decides. The trigger is **cNGN's share of inventory value**, not +an absolute USDC figure: the first version alerted on "idle USDC over $200" and fired on a balanced +book holding $310 USDC against $348 of cNGN, where converting would have made the imbalance worse. +A threshold that is wrong the first time it runs is one an operator learns to ignore. + +| condition | action | +| --- | --- | +| cNGN under `CNGN_FLOOR_USD` ($100) | **urgent** — the bid side is about to go dark | +| cNGN under `CNGN_MIN_SHARE` (35%) of inventory value | **rebalance** — convert some USDC | +| USDC within 20% of `HALT_NET_INVENTORY_USD` ($800) | noted in the message | + +`--alert` refuses to run without `ALERT_WEBHOOK_URL` rather than logging and exiting 0: an alert +path that reaches nobody while reporting success is the failure this repo keeps finding. + ## Not done yet -The **withdrawal leg**: sub 15 → this signer. Sub 15 is owned by the market maker's wallet, so -either this signer is authorised to withdraw from it or the loop goes through `POST /v1/withdrawals` -with an MM-signed request. Until then the loop is manual on that one step. +The **withdrawal leg** cannot be automated as things stand, and it is worth being precise about why. +Withdrawals pay out **only to the subaccount owner** — the action data carries just `(asset, amount)`, +with no recipient — and `assertWithdrawalPolicy` refuses any withdrawal whose signer is not the +owner (`session-key withdrawals are not supported`). So USDC leaving sub 15 lands at the market +maker's wallet, signed by the market maker's key, and no delegation to this signer is possible. + +That leaves the operator in the loop for one step: withdraw, then forward to this signer. `check` +exists to make that step reliably prompted rather than remembered. Automating it properly means the +market maker doing the withdrawal itself — it already holds the key and already knows when USDC is +piling up — which is a change to the Go service and a separate piece of work. + +Granting this service `kms:Sign` on the market maker's key would also work and should **not** be +done: KMS grants are not partial, so it would confer full market-maker authority — cancelling every +order, withdrawing everything — which is a wider blast radius than the executor separation this key +exists to preserve. diff --git a/services/rebalance/src/alert.ts b/services/rebalance/src/alert.ts new file mode 100644 index 0000000..f7505b4 --- /dev/null +++ b/services/rebalance/src/alert.ts @@ -0,0 +1,17 @@ +/** + * Posts to the ops webhook. + * + * Both keys on purpose: Slack reads `text`, Discord reads `content`. The settlement canary and the + * ops-box scripts post the same shape, so one webhook serves every sender. + */ +export type PostAlert = (url: string, text: string) => Promise; + +export const postAlert: PostAlert = async (url, text) => { + const response = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ text, content: text }), + signal: AbortSignal.timeout(10_000), + }); + if (!response.ok) throw new Error(`webhook returned ${response.status}`); +}; diff --git a/services/rebalance/src/check.ts b/services/rebalance/src/check.ts new file mode 100644 index 0000000..a568bf0 --- /dev/null +++ b/services/rebalance/src/check.ts @@ -0,0 +1,54 @@ +/** + * Reads the market maker's subaccount and says whether a rebalance is due. + * + * Meant for a timer. `--alert` posts to the ops webhook; without it this only prints, so the + * threshold can be tuned against live numbers without paging anyone. + */ +import { formatUnits } from 'viem'; +import { postAlert as defaultPostAlert, type PostAlert } from './alert.js'; +import type { Clients } from './clients.js'; +import type { Config } from './config.js'; +import { assessInventory } from './inventory.js'; +import { latestSnapshot, priceFromSnapshot } from './quote.js'; +import { CNGN, CNGN_ESCROW, LEDGER_DECIMALS, SUBACCOUNTS, SUBACCOUNTS_ABI, TOKEN_DECIMALS, USDC, USDC_ESCROW } from './venue.js'; + +export async function check( + config: Config, + clients: Clients, + alert: boolean, + post: PostAlert = defaultPostAlert, +): Promise { + const sub = config.MM_SUBACCOUNT_ID; + const [rows, snapshot] = await Promise.all([ + clients.publicClient.readContract({ address: SUBACCOUNTS, abi: SUBACCOUNTS_ABI, functionName: 'getAccountBalances', args: [sub] }), + latestSnapshot(config.INDEXER_URL, USDC, CNGN), + ]); + const held = (escrow: string) => rows.find((r) => r.asset.toLowerCase() === escrow.toLowerCase())?.balance ?? 0n; + + // One whole USDC, priced through the same path a rebalance would use, so the valuation and the + // trade cannot disagree. + const oneUsdc = 10n ** BigInt(TOKEN_DECIMALS); + const rate = priceFromSnapshot(snapshot, oneUsdc, config.MAX_SNAPSHOT_AGE_SECONDS).rate; + + const verdict = assessInventory({ usdc: held(USDC_ESCROW), cngn: held(CNGN_ESCROW), rate }, { + cngnMinShare: config.CNGN_MIN_SHARE, + cngnFloorUsd: config.CNGN_FLOOR_USD, + haltNetInventoryUsd: config.HALT_NET_INVENTORY_USD, + }, sub); + + console.log(`sub ${sub} USDC ${formatUnits(held(USDC_ESCROW), LEDGER_DECIMALS)} / cNGN ${formatUnits(held(CNGN_ESCROW), LEDGER_DECIMALS)}`); + console.log(`rate ${rate.toFixed(4)} (snapshot ${snapshot.snapshotTime.toISOString()})`); + console.log(`valued USDC $${verdict.usdcUsd.toFixed(2)} / cNGN $${verdict.cngnUsd.toFixed(2)} (cNGN ${(verdict.cngnShare * 100).toFixed(1)}%)`); + console.log(`action ${verdict.action}`); + for (const reason of verdict.reasons) console.log(` - ${reason}`); + + if (!alert) { if (verdict.action !== 'none') console.log('\n(pass --alert to post this to the ops webhook)'); return; } + if (verdict.action === 'none') return; + if (!config.ALERT_WEBHOOK_URL) { + // Refuse rather than log-and-exit-0: an alert path that reaches nobody while reporting success + // is the failure mode this repo keeps finding. + throw new Error('ALERT_WEBHOOK_URL is not set, so --alert would reach nobody'); + } + await post(config.ALERT_WEBHOOK_URL, verdict.message); + console.log('alert posted'); +} diff --git a/services/rebalance/src/cli.ts b/services/rebalance/src/cli.ts index 5f4eb4e..6600cc8 100644 --- a/services/rebalance/src/cli.ts +++ b/services/rebalance/src/cli.ts @@ -1,6 +1,7 @@ /** * cNGN rebalance CLI. Every command is a DRY RUN unless `--execute` is passed. * + * rebalance check [--alert] is a rebalance due? (for a timer) * rebalance quote [amount] what the live feed prices this at * rebalance approve [amount] exact-amount USDC allowance to the gateway * rebalance swap [amount] place, run the auction, fill @@ -14,12 +15,13 @@ import { formatUnits, parseUnits } from 'viem'; import { createClients } from './clients.js'; import { loadConfig } from './config.js'; import { cancel } from './cancel.js'; +import { check } from './check.js'; import { deposit } from './deposit.js'; import { latestSnapshot, priceFromSnapshot } from './quote.js'; import { approve, swap } from './swap.js'; import { CNGN, TOKEN_DECIMALS, USDC } from './venue.js'; -const USAGE = `usage: rebalance [amount|commitment] [--execute]`; +const USAGE = `usage: rebalance [amount|commitment] [--execute]`; async function main(): Promise { const argv = process.argv.slice(2); @@ -47,6 +49,7 @@ async function main(): Promise { case 'approve': return approve(config, clients, parseUnits(arg ?? '20', TOKEN_DECIMALS), execute); case 'swap': return swap(config, clients, parseUnits(arg ?? '20', TOKEN_DECIMALS), execute); case 'cancel': return cancel(config, clients, arg, execute); + case 'check': return check(config, clients, argv.includes('--alert')); case 'deposit': return deposit(config, clients, arg ? parseUnits(arg, TOKEN_DECIMALS) : undefined, execute); default: throw new Error(`unknown command "${command}"\n${USAGE}`); } diff --git a/services/rebalance/src/config.ts b/services/rebalance/src/config.ts index 6f6e89b..7b7a4f5 100644 --- a/services/rebalance/src/config.ts +++ b/services/rebalance/src/config.ts @@ -44,6 +44,25 @@ const schema = z.object({ DEADLINE_BLOCKS: z.coerce.bigint().default(120n), /** Auction window handed to executeBest. */ AUCTION_MS: z.coerce.number().int().positive().default(30_000), + /** Slack/Discord-compatible webhook the `check` command alerts to. */ + ALERT_WEBHOOK_URL: z.string().url().optional(), + /** + * Rebalance when cNGN falls below this share of total inventory value. + * + * A ratio, not an absolute USDC figure: the first version of this check alerted on idle USDC + * over $200 and fired on a balanced book ($310 USDC against $348 of cNGN), where converting + * would have worsened the imbalance. 0.35 leaves room for the ladder to lean either way + * without paging. + */ + CNGN_MIN_SHARE: z.coerce.number().positive().max(1).default(0.35), + /** + * cNGN holdings, valued in dollars, below which the bid side is close to dark. This is the + * condition that actually hurts -- idle USDC is only a pending problem, a thin cNGN side is a + * live one -- so it is treated as urgent rather than as a larger version of the same alert. + */ + CNGN_FLOOR_USD: z.coerce.number().positive().default(100), + /** MM_MAX_NET_INVENTORY, mirrored here only to say how close the halt is. */ + HALT_NET_INVENTORY_USD: z.coerce.number().positive().default(800), }); export type Config = z.infer & { bundlerUrl: string }; diff --git a/services/rebalance/src/inventory.test.ts b/services/rebalance/src/inventory.test.ts new file mode 100644 index 0000000..1fed5f9 --- /dev/null +++ b/services/rebalance/src/inventory.test.ts @@ -0,0 +1,65 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { assessInventory, type InventoryThresholds } from './inventory.js'; + +/** Subaccount balances are 18dp whatever the token's own decimals are. */ +const ledger = (units: number) => BigInt(Math.round(units * 1e6)) * 10n ** 12n; +const RATE = 1368.3155; +const thresholds: InventoryThresholds = { cngnMinShare: 0.35, cngnFloorUsd: 100, haltNetInventoryUsd: 800 }; +const SUB = 15n; + +test('a balanced, well-funded book asks for nothing', () => { + // Sub 15's real state on 2026-09-18: 310 USDC and 475,896 cNGN — $310 against $348, cNGN 53%. + // An earlier version of this check alerted here, on "idle USDC over $200", and would have asked + // for a conversion that made the imbalance worse. This test exists to keep that fixed. + const v = assessInventory({ usdc: ledger(310.002), cngn: ledger(475_896.28), rate: RATE }, thresholds, SUB); + assert.equal(v.action, 'none'); + assert.deepEqual(v.reasons, []); + assert.ok(v.cngnShare > 0.5, `expected cNGN over half, got ${v.cngnShare}`); +}); + +test('a book leaning to USDC is due a rebalance', () => { + // The shape one-directional flow produces: cNGN sold down, USDC piled up. + const v = assessInventory({ usdc: ledger(500), cngn: ledger(150_000), rate: RATE }, thresholds, SUB); + assert.equal(v.action, 'rebalance'); + assert.match(v.reasons.join(' '), /cNGN is 18% of inventory, under the 35% floor/); +}); + +test('a nearly dark bid side is urgent', () => { + const v = assessInventory({ usdc: ledger(10), cngn: ledger(50_000), rate: RATE }, thresholds, SUB); + assert.equal(v.action, 'urgent'); + assert.match(v.reasons.join(' '), /bids will go dark/); +}); + +test('share alone does not raise urgency', () => { + // Lopsided but with a cNGN side well above the floor: convert, do not page. + const v = assessInventory({ usdc: ledger(5000), cngn: ledger(500_000), rate: RATE }, thresholds, SUB); + assert.equal(v.action, 'rebalance'); +}); + +test('warns when USDC approaches the market maker halt', () => { + const v = assessInventory({ usdc: ledger(700), cngn: ledger(475_896), rate: RATE }, thresholds, SUB); + assert.match(v.reasons.join(' '), /within 20% of the market maker's \$800 inventory halt/); +}); + +test('an empty subaccount is empty, not lopsided', () => { + // Nothing here can fix an unfunded venue, and a 0/0 ratio must not page about it. + const v = assessInventory({ usdc: 0n, cngn: 0n, rate: RATE }, thresholds, SUB); + assert.equal(v.action, 'none'); + assert.deepEqual(v.reasons, []); +}); + +test('says what to do, because the next step cannot be automated', () => { + const v = assessInventory({ usdc: ledger(500), cngn: ledger(150_000), rate: RATE }, thresholds, SUB); + assert.match(v.message, /Withdraw USDC from the subaccount/); +}); + +test('refuses to value the cNGN side without a rate', () => { + assert.throws(() => assessInventory({ usdc: ledger(250), cngn: ledger(1000), rate: 0 }, thresholds, SUB), /rate must be positive/); +}); + +test('the cNGN side is valued at the rate, not counted in tokens', () => { + // 136,832 cNGN is a six-figure token balance and about $100 — the trap this valuation avoids. + const v = assessInventory({ usdc: ledger(0), cngn: ledger(136_832), rate: RATE }, thresholds, SUB); + assert.ok(Math.abs(v.cngnUsd - 100) < 1, `expected ~$100, got ${v.cngnUsd}`); +}); diff --git a/services/rebalance/src/inventory.ts b/services/rebalance/src/inventory.ts new file mode 100644 index 0000000..ed6172d --- /dev/null +++ b/services/rebalance/src/inventory.ts @@ -0,0 +1,87 @@ +/** + * Decides when the market maker needs a rebalance. + * + * The failure this watches for is the bid side going dark. The market maker's flow is + * one-directional -- it sells cNGN for USDC -- so cNGN drains while USDC piles up, and the first + * visible symptom is an order book with no bids. By then the venue has already stopped quoting one + * side, which is what this exists to get ahead of. + * + * Kept pure and separate from the reads so the thresholds are testable without a chain: the point + * of an alert is that it fires, and an alert nobody has watched fire is the kind this repo has + * repeatedly found not to work. + */ +import { formatUnits } from 'viem'; +import { LEDGER_DECIMALS } from './venue.js'; + +export type InventoryReading = { + /** Subaccount holdings, 18dp as SubAccounts reports them. */ + usdc: bigint; + cngn: bigint; + /** cNGN per USDC, from the live HyperFX feed — what a rebalance would actually convert at. */ + rate: number; +}; + +export type InventoryThresholds = { + /** + * Rebalance when cNGN's share of total inventory value falls below this fraction. + * + * A share rather than an absolute USDC figure, because absolute USDC is not the condition. The + * first version of this alerted on "idle USDC over $200" and fired on a subaccount holding $310 + * USDC against $348 of cNGN -- a balanced, well-funded book where converting more USDC would + * have made the imbalance worse. A threshold that is wrong the first time it runs is one an + * operator learns to ignore. + */ + cngnMinShare: number; + /** Below this much cNGN (valued in USD), the bid side is nearly dark whatever the share says. */ + cngnFloorUsd: number; + /** The market maker's own halt, for context in the message. Purely informational. */ + haltNetInventoryUsd?: number; +}; + +export type Verdict = { + action: 'none' | 'rebalance' | 'urgent'; + usdcUsd: number; + cngnUsd: number; + /** cNGN's share of total inventory value, 0..1. */ + cngnShare: number; + reasons: string[]; + message: string; +}; + +export function assessInventory(reading: InventoryReading, thresholds: InventoryThresholds, subaccount: bigint): Verdict { + if (!(reading.rate > 0)) throw new Error('rate must be positive to value the cNGN side'); + const usdcUsd = Number(formatUnits(reading.usdc, LEDGER_DECIMALS)); + const cngnUsd = Number(formatUnits(reading.cngn, LEDGER_DECIMALS)) / reading.rate; + + const total = usdcUsd + cngnUsd; + // An empty subaccount is not lopsided; it is empty, and nothing here can fix that. + const cngnShare = total > 0 ? cngnUsd / total : 1; + + const reasons: string[] = []; + // Urgent and routine are separate conditions, not a severity ladder on one number: a nearly dark + // bid side is a live problem, a drifting ratio is only a pending one. Both can be true at once. + const urgent = cngnUsd < thresholds.cngnFloorUsd && total > 0; + if (urgent) reasons.push(`cNGN side is $${cngnUsd.toFixed(0)}, under the $${thresholds.cngnFloorUsd} floor — bids will go dark`); + const routine = cngnShare < thresholds.cngnMinShare; + if (routine) { + reasons.push(`cNGN is ${(cngnShare * 100).toFixed(0)}% of inventory, under the ${(thresholds.cngnMinShare * 100).toFixed(0)}% floor`); + } + if (thresholds.haltNetInventoryUsd !== undefined && usdcUsd >= thresholds.haltNetInventoryUsd * 0.8) { + reasons.push(`within 20% of the market maker's $${thresholds.haltNetInventoryUsd} inventory halt`); + } + + const action: Verdict['action'] = urgent ? 'urgent' : routine ? 'rebalance' : 'none'; + const head = action === 'urgent' + ? `cNGN rebalance URGENT (sub ${subaccount})` + : action === 'rebalance' + ? `cNGN rebalance due (sub ${subaccount})` + : `cNGN inventory healthy (sub ${subaccount})`; + const balances = `USDC $${usdcUsd.toFixed(0)} / cNGN $${cngnUsd.toFixed(0)} (cNGN ${(cngnShare * 100).toFixed(0)}%) at ${reading.rate.toFixed(2)}`; + const body = reasons.length ? `${balances}. ${reasons.join('; ')}.` : `${balances}.`; + // The next step is a human one -- a withdrawal pays out only to the subaccount owner and cannot + // be delegated -- so the message says what to do rather than just what is true. + const next = action === 'none' + ? '' + : ' Withdraw USDC from the subaccount to the rebalance signer, then `pnpm rebalance swap --execute` and `pnpm rebalance deposit --execute`.'; + return { action, usdcUsd, cngnUsd, cngnShare, reasons, message: `${head}: ${body}${next}` }; +} diff --git a/services/rebalance/src/venue.ts b/services/rebalance/src/venue.ts index f3deadb..9380537 100644 --- a/services/rebalance/src/venue.ts +++ b/services/rebalance/src/venue.ts @@ -14,6 +14,8 @@ export const USDC = '0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913' as const; export const CNGN = '0x46C85152bFe9f96829aA94755D9f915F9B10EF5F' as const; /** WrappedERC20Asset escrow for cNGN — the venue's cNGN leg, and the spot market's asset_address. */ export const CNGN_ESCROW = '0x9D806fD040a719D27a8E5E77dc5aE0ED1e089493' as const; +/** WrappedERC20Asset escrow for USDC — the wrapped-quote module's quoteAsset(). */ +export const USDC_ESCROW = '0x364058aFF6f36E01505fB2Cc870f8B6BD4835e84' as const; export const SUBACCOUNTS = '0x7019244E25FA416e6Ca2ed2F3cA25277aef72843' as const; /** Both tokens are 6dp on Base. SubAccounts reports balances in 18dp regardless. */