From 1ae82ff28b5967e68f395ae5f192ca35286533fa Mon Sep 17 00:00:00 2001 From: githoboman Date: Mon, 29 Jun 2026 21:32:45 +0100 Subject: [PATCH 1/2] feat: implement contract liveness tracking with tombstone database schema and indexer utilities --- .../migration.sql | 17 ++ prisma/schema.prisma | 25 +++ src/__tests__/tombstones.test.ts | 177 ++++++++++++++++++ src/indexer/tombstones.ts | 177 ++++++++++++++++++ 4 files changed, 396 insertions(+) create mode 100644 prisma/migrations/20260629120000_add_contract_tombstones/migration.sql create mode 100644 src/__tests__/tombstones.test.ts create mode 100644 src/indexer/tombstones.ts diff --git a/prisma/migrations/20260629120000_add_contract_tombstones/migration.sql b/prisma/migrations/20260629120000_add_contract_tombstones/migration.sql new file mode 100644 index 00000000..3c726cd3 --- /dev/null +++ b/prisma/migrations/20260629120000_add_contract_tombstones/migration.sql @@ -0,0 +1,17 @@ +-- Track contract liveness: one tombstone row per contract whose persistent +-- storage instance entry has expired (liveUntilLedger fell behind the current +-- ledger). Downstream consumers watch this table for a "contract gone" signal. +CREATE TABLE "wraith"."ContractTombstone" ( + "id" SERIAL NOT NULL, + "contractId" TEXT NOT NULL, + "liveUntilLedger" INTEGER NOT NULL, + "detectedLedger" INTEGER NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "ContractTombstone_pkey" PRIMARY KEY ("id") +); + +-- One tombstone per contract; first expiry detection wins, re-detection is a no-op. +CREATE UNIQUE INDEX "ContractTombstone_contractId_key" ON "wraith"."ContractTombstone"("contractId"); + +CREATE INDEX "ContractTombstone_detectedLedger_idx" ON "wraith"."ContractTombstone"("detectedLedger"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 9b50798c..089ee611 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -214,6 +214,31 @@ model RetentionJobRun { @@schema("wraith") } +// ─── Contract Tombstones ──────────────────────────────────────────────────── +// A tombstone marks a contract whose persistent storage has expired (its +// instance entry's liveUntilLedger fell behind the current ledger). Downstream +// consumers watch this table for a "the contract is gone" signal. One row per +// contract — the first expiry detection wins; re-detection is idempotent. +model ContractTombstone { + id Int @id @default(autoincrement()) + + // The contract whose storage expired (C...) + contractId String @unique + + // The contract instance's liveUntilLedger at the moment of expiry — the last + // ledger for which the entry was still live. + liveUntilLedger Int + + // The ledger at which we observed the entry had expired + // (detectedLedger > liveUntilLedger). + detectedLedger Int + + createdAt DateTime @default(now()) + + @@index([detectedLedger]) + @@schema("wraith") +} + // ─── Backfill Cursor ─────────────────────────────────────────────────────────── // Durable cursor so the backfill job can resume mid-range after a crash. // Always row ID 1 — singleton, one backfill at a time. diff --git a/src/__tests__/tombstones.test.ts b/src/__tests__/tombstones.test.ts new file mode 100644 index 00000000..55fd2146 --- /dev/null +++ b/src/__tests__/tombstones.test.ts @@ -0,0 +1,177 @@ +/** + * Tests for contract tombstones — liveness tracking + expiry detection. + * + * Covers the pure expiry rule, the injectable TTL fetcher / detection helpers, + * and the idempotent insert path. No network is used — a fake TTL fetcher is + * injected throughout — and the Prisma client is mocked for the persistence + * tests, mirroring the sac-detect test style. + */ + +import { describe, it, expect, jest, beforeEach } from "@jest/globals"; + +// Mock the Prisma client before importing the module under test so the +// `insertTombstones` path exercises a fake `createMany`. +const createMany = jest.fn< + (args: { data: unknown[]; skipDuplicates: boolean }) => Promise<{ count: number }> +>(); +jest.mock("../db", () => ({ + prisma: { contractTombstone: { createMany } }, +})); + +import { + isExpired, + tombstoneFor, + fetchLiveness, + detectExpiredContracts, + insertTombstones, + tombstoneExpiredContracts, + type TtlFetcher, + type ContractLiveness, + type TombstoneRecord, +} from "../indexer/tombstones"; + +// ─── Fixture ────────────────────────────────────────────────────────────────── +// A small fleet of contracts with known TTLs, and the "current" ledger we +// evaluate them against. CALIVE is comfortably live; CEXPIRED expired one ledger +// ago; CBORDER's liveUntil equals the current ledger (still live, edge case); +// CUNKNOWN has no resolvable TTL. +const CURRENT_LEDGER = 1_000; + +const TTL_FIXTURE: Record = { + CALIVE: 5_000, + CEXPIRED: 999, + CBORDER: 1_000, + CUNKNOWN: null, +}; + +const fixtureFetcher: TtlFetcher = async (contractId) => + contractId in TTL_FIXTURE ? TTL_FIXTURE[contractId] : null; + +beforeEach(() => { + createMany.mockReset(); +}); + +describe("isExpired", () => { + it("is false while the current ledger is before liveUntil", () => { + expect(isExpired(5_000, 1_000)).toBe(false); + }); + + it("is false on the exact liveUntil ledger (live through that ledger)", () => { + expect(isExpired(1_000, 1_000)).toBe(false); + }); + + it("is true once the current ledger passes liveUntil", () => { + expect(isExpired(999, 1_000)).toBe(true); + }); + + it("never treats an unknown TTL as expired", () => { + expect(isExpired(null, 1_000)).toBe(false); + }); +}); + +describe("tombstoneFor", () => { + it("returns a tombstone for an expired contract", () => { + const liveness: ContractLiveness = { contractId: "CEXPIRED", liveUntilLedger: 999 }; + expect(tombstoneFor(liveness, CURRENT_LEDGER)).toEqual({ + contractId: "CEXPIRED", + liveUntilLedger: 999, + detectedLedger: CURRENT_LEDGER, + }); + }); + + it("returns null for a live contract", () => { + expect(tombstoneFor({ contractId: "CALIVE", liveUntilLedger: 5_000 }, CURRENT_LEDGER)).toBeNull(); + }); + + it("returns null when the TTL is unknown", () => { + expect(tombstoneFor({ contractId: "CUNKNOWN", liveUntilLedger: null }, CURRENT_LEDGER)).toBeNull(); + }); +}); + +describe("fetchLiveness", () => { + it("resolves liveness via the injected fetcher", async () => { + const liveness = await fetchLiveness(["CALIVE", "CEXPIRED"], fixtureFetcher); + expect(liveness).toEqual([ + { contractId: "CALIVE", liveUntilLedger: 5_000 }, + { contractId: "CEXPIRED", liveUntilLedger: 999 }, + ]); + }); + + it("de-duplicates contract IDs so each is fetched once", async () => { + const fetcher = jest.fn(async () => 5_000); + await fetchLiveness(["CA", "CA", "CB"], fetcher); + expect(fetcher).toHaveBeenCalledTimes(2); + }); +}); + +describe("detectExpiredContracts", () => { + it("emits tombstones only for contracts past their liveUntil", async () => { + const tombstones = await detectExpiredContracts( + ["CALIVE", "CEXPIRED", "CBORDER", "CUNKNOWN"], + CURRENT_LEDGER, + fixtureFetcher, + ); + + expect(tombstones).toEqual([ + { contractId: "CEXPIRED", liveUntilLedger: 999, detectedLedger: CURRENT_LEDGER }, + ]); + }); + + it("returns an empty array when every contract is still live", async () => { + const tombstones = await detectExpiredContracts(["CALIVE", "CBORDER"], CURRENT_LEDGER, fixtureFetcher); + expect(tombstones).toEqual([]); + }); +}); + +describe("insertTombstones", () => { + it("inserts on expiry detection, idempotently by contractId", async () => { + createMany.mockResolvedValue({ count: 1 }); + + const records: TombstoneRecord[] = [ + { contractId: "CEXPIRED", liveUntilLedger: 999, detectedLedger: CURRENT_LEDGER }, + ]; + const inserted = await insertTombstones(records); + + expect(inserted).toBe(1); + expect(createMany).toHaveBeenCalledWith({ data: records, skipDuplicates: true }); + }); + + it("skips the DB entirely for an empty batch", async () => { + const inserted = await insertTombstones([]); + expect(inserted).toBe(0); + expect(createMany).not.toHaveBeenCalled(); + }); +}); + +describe("tombstoneExpiredContracts", () => { + it("detects expiry and persists exactly the expired contracts", async () => { + createMany.mockResolvedValue({ count: 1 }); + + const { tombstones, inserted } = await tombstoneExpiredContracts( + ["CALIVE", "CEXPIRED", "CUNKNOWN"], + CURRENT_LEDGER, + fixtureFetcher, + ); + + expect(tombstones).toEqual([ + { contractId: "CEXPIRED", liveUntilLedger: 999, detectedLedger: CURRENT_LEDGER }, + ]); + expect(inserted).toBe(1); + expect(createMany).toHaveBeenCalledWith({ + data: [{ contractId: "CEXPIRED", liveUntilLedger: 999, detectedLedger: CURRENT_LEDGER }], + skipDuplicates: true, + }); + }); + + it("writes nothing when no contract has expired", async () => { + const { tombstones, inserted } = await tombstoneExpiredContracts( + ["CALIVE", "CBORDER"], + CURRENT_LEDGER, + fixtureFetcher, + ); + + expect(tombstones).toEqual([]); + expect(inserted).toBe(0); + expect(createMany).not.toHaveBeenCalled(); + }); +}); diff --git a/src/indexer/tombstones.ts b/src/indexer/tombstones.ts new file mode 100644 index 00000000..680bc322 --- /dev/null +++ b/src/indexer/tombstones.ts @@ -0,0 +1,177 @@ +/** + * Contract tombstones — liveness tracking for expired contract storage. + * + * Soroban persistent state is not permanent: every contract's instance ledger + * entry carries a `liveUntilLedger`. Once the network's current ledger passes + * that value the entry is archived and the contract effectively disappears — + * its storage can no longer be read without a restore. Downstream consumers + * that cached a contract's events need a signal that this has happened, so when + * we observe an expiry we emit a *tombstone* row. + * + * Detection strategy (mirrors sac-detect's instance lookup, #136): + * 1. Read the contract instance ledger entry (ContractData keyed by + * `ScVal::LedgerKeyContractInstance`) and pull its `liveUntilLedgerSeq`. + * 2. Compare against the current ledger: expired when current > liveUntil. + * 3. Insert one tombstone per contract, idempotently — the first detection + * wins; re-observing the same expiry is a no-op. + * + * The TTL fetcher is injectable so the detection logic can be exercised without + * a network round-trip. + */ + +import { xdr } from "@stellar/stellar-sdk"; +import { getRpc } from "../rpc"; +import { prisma } from "../db"; + +// ─── Types ──────────────────────────────────────────────────────────────────── + +/** + * Liveness of a single contract's instance entry at a point in time. + * `liveUntilLedger` is null when the entry has no TTL / could not be resolved. + */ +export interface ContractLiveness { + contractId: string; + liveUntilLedger: number | null; +} + +/** A tombstone ready to be persisted. */ +export interface TombstoneRecord { + contractId: string; + liveUntilLedger: number; + detectedLedger: number; +} + +// ─── Pure helpers ───────────────────────────────────────────────────────────── + +/** + * True when a contract instance has expired relative to `currentLedger`. + * + * An entry is live *through* its `liveUntilLedger`, so it is only expired once + * the current ledger has moved strictly past it. A null TTL (unknown / no entry) + * is never treated as expired — we don't tombstone on missing information. + */ +export function isExpired( + liveUntilLedger: number | null, + currentLedger: number, +): boolean { + if (liveUntilLedger === null) return false; + return currentLedger > liveUntilLedger; +} + +/** + * Build a tombstone for an expired contract, or null if it is still live (or its + * TTL is unknown). Keeping this pure makes the expiry rule trivially testable. + */ +export function tombstoneFor( + liveness: ContractLiveness, + currentLedger: number, +): TombstoneRecord | null { + if (!isExpired(liveness.liveUntilLedger, currentLedger)) return null; + return { + contractId: liveness.contractId, + liveUntilLedger: liveness.liveUntilLedger as number, + detectedLedger: currentLedger, + }; +} + +// ─── TTL fetch ──────────────────────────────────────────────────────────────── + +/** + * Resolve the `liveUntilLedger` of a contract's instance entry, or null if the + * contract has no instance entry / the lookup fails. Injectable for testing. + */ +export type TtlFetcher = (contractId: string) => Promise; + +async function fetchLiveUntilLedger(contractId: string): Promise { + try { + const entry = await getRpc().getContractData( + contractId, + xdr.ScVal.scvLedgerKeyContractInstance(), + ); + return entry.liveUntilLedgerSeq ?? null; + } catch { + // Missing entry, already-evicted state, or RPC error — not determinable. + return null; + } +} + +/** + * Read liveness for a set of contracts, de-duplicating IDs so each unique + * contract is looked up at most once. One RPC call per unique contract. + */ +export async function fetchLiveness( + contractIds: Iterable, + fetchTtl: TtlFetcher = fetchLiveUntilLedger, +): Promise { + const unique = [...new Set(contractIds)]; + return Promise.all( + unique.map(async (contractId) => ({ + contractId, + liveUntilLedger: await fetchTtl(contractId), + })), + ); +} + +// ─── Detection ────────────────────────────────────────────────────────────── + +/** + * Compute the tombstones owed for a batch of contracts at `currentLedger`, + * fetching each one's TTL. Pure-ish: does no DB writes, so callers can inspect + * or test the result before persisting. + */ +export async function detectExpiredContracts( + contractIds: Iterable, + currentLedger: number, + fetchTtl: TtlFetcher = fetchLiveUntilLedger, +): Promise { + const liveness = await fetchLiveness(contractIds, fetchTtl); + return liveness + .map((l) => tombstoneFor(l, currentLedger)) + .filter((t): t is TombstoneRecord => t !== null); +} + +// ─── Persistence ────────────────────────────────────────────────────────────── + +/** + * Idempotently insert tombstone rows. Conflicts on `contractId` are ignored — + * the first expiry detection wins, so replaying a ledger range never duplicates + * or overwrites a contract's tombstone. Returns the number of rows inserted. + */ +export async function insertTombstones( + records: TombstoneRecord[], +): Promise { + if (records.length === 0) return 0; + + const result = await prisma.contractTombstone.createMany({ + data: records, + skipDuplicates: true, + }); + + return result.count; +} + +/** + * Detect expired contracts at `currentLedger` and persist a tombstone for each. + * Combines {@link detectExpiredContracts} and {@link insertTombstones}; returns + * the records and how many were newly inserted (vs. already tombstoned). + */ +export async function tombstoneExpiredContracts( + contractIds: Iterable, + currentLedger: number, + fetchTtl: TtlFetcher = fetchLiveUntilLedger, +): Promise<{ tombstones: TombstoneRecord[]; inserted: number }> { + const tombstones = await detectExpiredContracts( + contractIds, + currentLedger, + fetchTtl, + ); + const inserted = await insertTombstones(tombstones); + + if (inserted > 0) { + console.log( + `[tombstone] ${inserted} contract(s) tombstoned at ledger ${currentLedger}`, + ); + } + + return { tombstones, inserted }; +} From 5505a2bb7d9de683cac174bbf001cee54b2a2ff1 Mon Sep 17 00:00:00 2001 From: Miracle656 Date: Tue, 1 Sep 2026 18:35:24 +0100 Subject: [PATCH 2/2] Wire the tombstone detector into the indexer loop, and scope it per network MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The detection logic was complete and well tested but nothing ever called it: `detectExpiredContracts` / `tombstoneExpiredContracts` were referenced only by their own tests, so the migration added a `ContractTombstone` table that would have stayed permanently empty. That failure is invisible — a detector that never runs looks exactly like a chain on which nothing has expired. Wiring: - `maybeTombstoneExpiredContracts(loop, currentLedger)` runs inside the poll loop on its own cadence, `TOMBSTONE_CHECK_EVERY_CYCLES` (default 100 ≈ 10 min at a 6s poll). Each check costs one RPC call per unique watched contract, and a contract TTL is measured in weeks, so checking every cycle would multiply the RPC budget by the size of the watch list for no benefit. - Its own counter, separate from `pollCycleCount`. Sharing one would make whichever cadence is shorter starve the other, since the prune resets it. - Failures are caught, not propagated: a missed liveness check is retried next cadence, whereas a loop that dies on an RPC hiccup stops indexing entirely. - Extracted as an exported function rather than left inline, so the wiring is assertable. Six tests cover it, including that the check recurs rather than firing once and that a failed check does not stall the loop. Network scoping, which this branch predates: - `ContractTombstone` gains a `network` column, `@@unique([network, contractId])` instead of a global unique on `contractId`. The same contract id exists on both chains with different TTLs, so a global unique would let a testnet expiry permanently suppress the mainnet tombstone. - The migration is renumbered to 20260901120000, after 20260829120000_add_network. That migration back-filled `network` onto the tables that existed when it was written; this one did not, so ordering it earlier would have left ContractTombstone as the only network-blind table in the schema. - `fetchLiveUntilLedger` now calls `getRpc(network)`. Reading a mainnet contract's TTL off the testnet RPC fails the lookup, returns null, and the "unknown TTL is never expired" guard then means the mainnet loop silently never tombstones anything. A test pins that the fetcher is asked for the same network the row is tagged with. tsc clean; full suite 351 passed. --- .env.example | 8 ++ .../migration.sql | 17 --- .../migration.sql | 26 +++++ prisma/schema.prisma | 12 ++- src/__tests__/tombstoneWiring.test.ts | 102 ++++++++++++++++++ src/__tests__/tombstones.test.ts | 47 +++++++- src/indexer.ts | 60 +++++++++++ src/indexer/tombstones.ts | 35 ++++-- 8 files changed, 277 insertions(+), 30 deletions(-) delete mode 100644 prisma/migrations/20260629120000_add_contract_tombstones/migration.sql create mode 100644 prisma/migrations/20260901120000_add_contract_tombstones/migration.sql create mode 100644 src/__tests__/tombstoneWiring.test.ts diff --git a/.env.example b/.env.example index 83c2b360..e9ed39c5 100644 --- a/.env.example +++ b/.env.example @@ -108,3 +108,11 @@ CACHE_KEY_PREFIX="wraith:cache:" # Per-route TTLs in milliseconds. CACHE_TTL_POPULAR_MS=60000 CACHE_TTL_SEARCH_MS=15000 + +# ─── Contract tombstones ──────────────────────────────────────────────────── +# How many poll cycles between contract-liveness checks (#137). Each check +# costs one RPC call per unique watched contract, so this is deliberately far +# rarer than a poll: a contract TTL is measured in weeks, and noticing an +# expiry ten minutes late costs nothing. Set to 0 to disable the check. +# Default: 100 cycles (~10 min at the default 6s poll interval). +TOMBSTONE_CHECK_EVERY_CYCLES=100 diff --git a/prisma/migrations/20260629120000_add_contract_tombstones/migration.sql b/prisma/migrations/20260629120000_add_contract_tombstones/migration.sql deleted file mode 100644 index 3c726cd3..00000000 --- a/prisma/migrations/20260629120000_add_contract_tombstones/migration.sql +++ /dev/null @@ -1,17 +0,0 @@ --- Track contract liveness: one tombstone row per contract whose persistent --- storage instance entry has expired (liveUntilLedger fell behind the current --- ledger). Downstream consumers watch this table for a "contract gone" signal. -CREATE TABLE "wraith"."ContractTombstone" ( - "id" SERIAL NOT NULL, - "contractId" TEXT NOT NULL, - "liveUntilLedger" INTEGER NOT NULL, - "detectedLedger" INTEGER NOT NULL, - "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - - CONSTRAINT "ContractTombstone_pkey" PRIMARY KEY ("id") -); - --- One tombstone per contract; first expiry detection wins, re-detection is a no-op. -CREATE UNIQUE INDEX "ContractTombstone_contractId_key" ON "wraith"."ContractTombstone"("contractId"); - -CREATE INDEX "ContractTombstone_detectedLedger_idx" ON "wraith"."ContractTombstone"("detectedLedger"); diff --git a/prisma/migrations/20260901120000_add_contract_tombstones/migration.sql b/prisma/migrations/20260901120000_add_contract_tombstones/migration.sql new file mode 100644 index 00000000..a6936c78 --- /dev/null +++ b/prisma/migrations/20260901120000_add_contract_tombstones/migration.sql @@ -0,0 +1,26 @@ +-- Track contract liveness: one tombstone row per contract whose persistent +-- storage instance entry has expired (liveUntilLedger fell behind the current +-- ledger). Downstream consumers watch this table for a "contract gone" signal. +-- +-- Ordered AFTER 20260829120000_add_network deliberately. That migration +-- back-filled a `network` column onto the tables that existed when it was +-- written; this table did not, so it carries its own from the start rather +-- than being silently left as the one network-blind table in the schema. +CREATE TABLE "wraith"."ContractTombstone" ( + "id" SERIAL NOT NULL, + "network" TEXT NOT NULL DEFAULT 'testnet', + "contractId" TEXT NOT NULL, + "liveUntilLedger" INTEGER NOT NULL, + "detectedLedger" INTEGER NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "ContractTombstone_pkey" PRIMARY KEY ("id") +); + +-- One tombstone per contract PER NETWORK; first expiry detection wins and +-- re-detection is a no-op. Scoping by network matters: the same contract id can +-- exist on both chains with different TTLs, and a global unique would let a +-- testnet expiry permanently suppress the mainnet tombstone. +CREATE UNIQUE INDEX "ContractTombstone_network_contractId_key" ON "wraith"."ContractTombstone"("network", "contractId"); + +CREATE INDEX "ContractTombstone_network_detectedLedger_idx" ON "wraith"."ContractTombstone"("network", "detectedLedger"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 7ccf6220..617bbd5e 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -266,8 +266,14 @@ model RetentionJobRun { model ContractTombstone { id Int @id @default(autoincrement()) + // Which chain the contract expired on. The same contract id can exist on both + // networks with different TTLs, so a tombstone is only meaningful per network + // — a global unique on contractId would let a testnet expiry suppress the + // mainnet one forever. + network String @default("testnet") + // The contract whose storage expired (C...) - contractId String @unique + contractId String // The contract instance's liveUntilLedger at the moment of expiry — the last // ledger for which the entry was still live. @@ -279,7 +285,9 @@ model ContractTombstone { createdAt DateTime @default(now()) - @@index([detectedLedger]) + // First expiry detection wins, per network; re-detection is a no-op. + @@unique([network, contractId]) + @@index([network, detectedLedger]) @@schema("wraith") } diff --git a/src/__tests__/tombstoneWiring.test.ts b/src/__tests__/tombstoneWiring.test.ts new file mode 100644 index 00000000..c93d671a --- /dev/null +++ b/src/__tests__/tombstoneWiring.test.ts @@ -0,0 +1,102 @@ +/** + * The tombstone detector is wired into the indexer loop (#137). + * + * These exist because the original failure here was invisible: the detection + * logic was complete, correct and thoroughly unit-tested, and nothing ever + * called it. A detector that never runs is indistinguishable from a chain on + * which nothing has expired — the table simply stays empty forever and no test + * goes red. So these assert the *call*, not the arithmetic. + */ + +import { describe, it, expect, jest, beforeEach } from "@jest/globals"; + +const tombstoneExpiredContracts = jest.fn< + (...args: unknown[]) => Promise<{ tombstones: unknown[]; inserted: number }> +>(); + +jest.mock("../indexer/tombstones", () => ({ tombstoneExpiredContracts })); + +import { maybeTombstoneExpiredContracts } from "../indexer"; + +const CURRENT_LEDGER = 5_000_000; + +/** Minimal LoopState stand-in — only the fields the helper reads. */ +function loopState(overrides: Record = {}) { + return { + network: "testnet", + allContractIds: ["CAAA", "CBBB"], + tombstoneCycleCount: 0, + ...overrides, + } as never; +} + +/** The cadence the helper is compiled with (TOMBSTONE_CHECK_EVERY_CYCLES). */ +const EVERY = parseInt(process.env.TOMBSTONE_CHECK_EVERY_CYCLES ?? "100", 10); + +describe("maybeTombstoneExpiredContracts", () => { + beforeEach(() => { + tombstoneExpiredContracts.mockReset(); + tombstoneExpiredContracts.mockResolvedValue({ tombstones: [], inserted: 0 }); + }); + + it("does not check on every poll — that would cost one RPC per contract per cycle", async () => { + const loop = loopState(); + + const ran = await maybeTombstoneExpiredContracts(loop, CURRENT_LEDGER); + + expect(ran).toBe(false); + expect(tombstoneExpiredContracts).not.toHaveBeenCalled(); + }); + + it("checks once the cadence is reached, and passes the watched contracts", async () => { + const loop = loopState({ tombstoneCycleCount: EVERY - 1 }); + + const ran = await maybeTombstoneExpiredContracts(loop, CURRENT_LEDGER); + + expect(ran).toBe(true); + expect(tombstoneExpiredContracts).toHaveBeenCalledTimes(1); + const [contractIds, ledger] = tombstoneExpiredContracts.mock.calls[0]; + expect(contractIds).toEqual(["CAAA", "CBBB"]); + expect(ledger).toBe(CURRENT_LEDGER); + }); + + it("passes the loop's own network, so each loop asks its own chain", async () => { + const loop = loopState({ network: "mainnet", tombstoneCycleCount: EVERY - 1 }); + + await maybeTombstoneExpiredContracts(loop, CURRENT_LEDGER); + + expect(tombstoneExpiredContracts.mock.calls[0][3]).toBe("mainnet"); + }); + + it("resets its counter so the check recurs rather than firing once forever", async () => { + const loop = loopState({ tombstoneCycleCount: EVERY - 1 }) as unknown as { + tombstoneCycleCount: number; + }; + + await maybeTombstoneExpiredContracts(loop as never, CURRENT_LEDGER); + expect(loop.tombstoneCycleCount).toBe(0); + + await maybeTombstoneExpiredContracts(loop as never, CURRENT_LEDGER); + expect(tombstoneExpiredContracts).toHaveBeenCalledTimes(1); + }); + + it("skips the RPC round-trip entirely when nothing is being watched", async () => { + const loop = loopState({ allContractIds: [], tombstoneCycleCount: EVERY - 1 }); + + const ran = await maybeTombstoneExpiredContracts(loop, CURRENT_LEDGER); + + expect(ran).toBe(false); + expect(tombstoneExpiredContracts).not.toHaveBeenCalled(); + }); + + it("swallows a failed check rather than stalling the indexer", async () => { + // A missed liveness check is retried on the next cadence. A loop that dies + // on an RPC hiccup is not self-healing, and stops indexing entirely. + tombstoneExpiredContracts.mockRejectedValue(new Error("rpc down")); + const loop = loopState({ tombstoneCycleCount: EVERY - 1 }); + + await expect( + maybeTombstoneExpiredContracts(loop, CURRENT_LEDGER), + ).resolves.toBe(true); + }); +}); diff --git a/src/__tests__/tombstones.test.ts b/src/__tests__/tombstones.test.ts index 55fd2146..635ed2b7 100644 --- a/src/__tests__/tombstones.test.ts +++ b/src/__tests__/tombstones.test.ts @@ -133,7 +133,10 @@ describe("insertTombstones", () => { const inserted = await insertTombstones(records); expect(inserted).toBe(1); - expect(createMany).toHaveBeenCalledWith({ data: records, skipDuplicates: true }); + expect(createMany).toHaveBeenCalledWith({ + data: [{ ...records[0], network: "testnet" }], + skipDuplicates: true, + }); }); it("skips the DB entirely for an empty batch", async () => { @@ -158,7 +161,14 @@ describe("tombstoneExpiredContracts", () => { ]); expect(inserted).toBe(1); expect(createMany).toHaveBeenCalledWith({ - data: [{ contractId: "CEXPIRED", liveUntilLedger: 999, detectedLedger: CURRENT_LEDGER }], + data: [ + { + contractId: "CEXPIRED", + liveUntilLedger: 999, + detectedLedger: CURRENT_LEDGER, + network: "testnet", + }, + ], skipDuplicates: true, }); }); @@ -175,3 +185,36 @@ describe("tombstoneExpiredContracts", () => { expect(createMany).not.toHaveBeenCalled(); }); }); + +describe("network scoping", () => { + it("stamps the tombstone with the network it was detected on", async () => { + createMany.mockResolvedValue({ count: 1 }); + + await tombstoneExpiredContracts( + ["CEXPIRED"], + CURRENT_LEDGER, + fixtureFetcher, + "mainnet", + ); + + const [args] = createMany.mock.calls[0] as [{ data: Array<{ network: string }> }]; + expect(args.data[0].network).toBe("mainnet"); + }); + + it("asks the TTL fetcher for the same network it will tag the row with", async () => { + // The bug this prevents: reading a mainnet contract's TTL off the testnet + // RPC. The lookup fails, fetchLiveUntilLedger returns null, isExpired + // treats null as "not expired" — and the mainnet loop silently never + // tombstones anything. No error, no row, nothing to notice. + createMany.mockResolvedValue({ count: 0 }); + const seen: Array = []; + const spy: TtlFetcher = async (_id, network) => { + seen.push(network); + return null; + }; + + await tombstoneExpiredContracts(["CANY"], CURRENT_LEDGER, spy, "mainnet"); + + expect(seen).toEqual(["mainnet"]); + }); +}); diff --git a/src/indexer.ts b/src/indexer.ts index d13ae3b1..5cf384cc 100644 --- a/src/indexer.ts +++ b/src/indexer.ts @@ -15,6 +15,7 @@ import { emitTransfer, emitHostFnLog } from "./events"; import { parseHostFnEvent, upsertHostFnLogs, type HostFnRecord } from "./indexer/host-fn-log"; import { tagSacTransfers } from "./indexer/sac-detect"; import { pollParallel } from "./indexer/parallel"; +import { tombstoneExpiredContracts } from "./indexer/tombstones"; import { isNftTransferEvent, parseNftEvents, fetchNftMetadata } from "./ingester/nft"; import { createSourceSwitcherWithConfig, type SourceSwitcher } from "./indexer/sources"; import { currentNetwork, enabledNetworks, resolveNetwork, type Network } from "./network"; @@ -98,6 +99,16 @@ const TIP_LAG = 2; // Prune old data every ~1 hour (600 poll cycles × 6s = 3600s) const PRUNE_EVERY_CYCLES = 600; +// How often to check watched contracts for expired storage (#137). Each check +// costs one RPC call per unique watched contract, so this is deliberately far +// rarer than a poll: a contract's TTL is measured in weeks, and detecting an +// expiry ten minutes late costs nothing. Doing it every cycle would multiply +// the indexer's RPC budget by the size of the watch list for no benefit. +const TOMBSTONE_EVERY_CYCLES = parseInt( + process.env.TOMBSTONE_CHECK_EVERY_CYCLES ?? "100", + 10, +); + // ─── Per-network loop state ─────────────────────────────────────────────────── /** * Everything one indexer loop owns. @@ -120,6 +131,12 @@ type LoopState = { startedAt: number; totalIndexed: number; pollCycleCount: number; + /** + * Separate from pollCycleCount, which the prune resets on its own schedule. + * Sharing one counter would make whichever cadence is shorter starve the + * other — the longer job would never reach its threshold. + */ + tombstoneCycleCount: number; }; const loops = new Map(); @@ -149,6 +166,7 @@ function createLoopState(network: Network): LoopState { startedAt: Date.now(), totalIndexed: 0, pollCycleCount: 0, + tombstoneCycleCount: 0, }; } @@ -208,6 +226,46 @@ function recordLedgerProgress(network: Network, fromLedger: number, highestLedge lastIndexedLedger.set({ network }, highestLedger); } +/** + * Periodic contract-liveness check (#137), run on its own cadence inside the + * poll loop. + * + * Soroban persistent storage is not permanent: once the ledger passes a + * contract instance's `liveUntilLedger` the entry is archived and the contract + * effectively disappears. Downstream consumers that cached its events need that + * signal, so we record a tombstone the first time we observe it. + * + * Exported so the wiring is testable. The check itself is cheap to get wrong in + * a way nothing notices — a detector that never runs looks exactly like a chain + * on which nothing has expired. + */ +export async function maybeTombstoneExpiredContracts( + loop: LoopState, + currentLedger: number, +): Promise { + loop.tombstoneCycleCount++; + + if (TOMBSTONE_EVERY_CYCLES <= 0) return false; + if (loop.tombstoneCycleCount < TOMBSTONE_EVERY_CYCLES) return false; + if (loop.allContractIds.length === 0) return false; + + loop.tombstoneCycleCount = 0; + + // Caught rather than allowed to reject: an RPC hiccup during a liveness + // check must not stall indexing. A missed check is retried next cycle; a + // stalled indexer is not self-healing. + await tombstoneExpiredContracts( + loop.allContractIds, + currentLedger, + undefined, + loop.network, + ).catch((e: unknown) => + console.error(`[indexer/${loop.network}] Tombstone check failed:`, e) + ); + + return true; +} + // ─── Core poll step ─────────────────────────────────────────────────────────── /** * Fetch one batch of events starting from `fromLedger`, parse and persist them. @@ -398,6 +456,8 @@ export async function startIndexer(network?: Network): Promise { console.error(`[indexer/${net}] Prune failed:`, e) ); } + + await maybeTombstoneExpiredContracts(loop, currentLedger); } catch (err) { console.error(`[indexer/${net}] Unhandled error in poll loop:`, err); // Back off before retrying to avoid hammering the RPC on persistent errors diff --git a/src/indexer/tombstones.ts b/src/indexer/tombstones.ts index 680bc322..f0e5ff63 100644 --- a/src/indexer/tombstones.ts +++ b/src/indexer/tombstones.ts @@ -22,6 +22,7 @@ import { xdr } from "@stellar/stellar-sdk"; import { getRpc } from "../rpc"; import { prisma } from "../db"; +import { resolveNetwork, type Network } from "../network"; // ─── Types ──────────────────────────────────────────────────────────────────── @@ -80,11 +81,20 @@ export function tombstoneFor( * Resolve the `liveUntilLedger` of a contract's instance entry, or null if the * contract has no instance entry / the lookup fails. Injectable for testing. */ -export type TtlFetcher = (contractId: string) => Promise; - -async function fetchLiveUntilLedger(contractId: string): Promise { +export type TtlFetcher = ( + contractId: string, + network?: Network, +) => Promise; + +async function fetchLiveUntilLedger( + contractId: string, + network?: Network, +): Promise { try { - const entry = await getRpc().getContractData( + // getRpc(network), not getRpc(): each loop must ask its own chain. Reading + // a mainnet contract's TTL off the testnet RPC would report it missing and, + // via the null guard below, silently never tombstone anything. + const entry = await getRpc(resolveNetwork(network)).getContractData( contractId, xdr.ScVal.scvLedgerKeyContractInstance(), ); @@ -102,12 +112,13 @@ async function fetchLiveUntilLedger(contractId: string): Promise export async function fetchLiveness( contractIds: Iterable, fetchTtl: TtlFetcher = fetchLiveUntilLedger, + network?: Network, ): Promise { const unique = [...new Set(contractIds)]; return Promise.all( unique.map(async (contractId) => ({ contractId, - liveUntilLedger: await fetchTtl(contractId), + liveUntilLedger: await fetchTtl(contractId, network), })), ); } @@ -123,8 +134,9 @@ export async function detectExpiredContracts( contractIds: Iterable, currentLedger: number, fetchTtl: TtlFetcher = fetchLiveUntilLedger, + network?: Network, ): Promise { - const liveness = await fetchLiveness(contractIds, fetchTtl); + const liveness = await fetchLiveness(contractIds, fetchTtl, network); return liveness .map((l) => tombstoneFor(l, currentLedger)) .filter((t): t is TombstoneRecord => t !== null); @@ -139,11 +151,13 @@ export async function detectExpiredContracts( */ export async function insertTombstones( records: TombstoneRecord[], + network?: Network, ): Promise { if (records.length === 0) return 0; + const net = resolveNetwork(network); const result = await prisma.contractTombstone.createMany({ - data: records, + data: records.map((record) => ({ ...record, network: net })), skipDuplicates: true, }); @@ -159,17 +173,20 @@ export async function tombstoneExpiredContracts( contractIds: Iterable, currentLedger: number, fetchTtl: TtlFetcher = fetchLiveUntilLedger, + network?: Network, ): Promise<{ tombstones: TombstoneRecord[]; inserted: number }> { + const net = resolveNetwork(network); const tombstones = await detectExpiredContracts( contractIds, currentLedger, fetchTtl, + net, ); - const inserted = await insertTombstones(tombstones); + const inserted = await insertTombstones(tombstones, net); if (inserted > 0) { console.log( - `[tombstone] ${inserted} contract(s) tombstoned at ledger ${currentLedger}`, + `[tombstone/${net}] ${inserted} contract(s) tombstoned at ledger ${currentLedger}`, ); }