From 3f3323bc24170ab4df82640d7be7f3da662a7dde Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:09:16 +0530 Subject: [PATCH 01/14] feat(indexer): scaffold chain-indexer app with sandbox sync tracer bullet --- apps/chain-indexer/README.md | 50 +++ apps/chain-indexer/drizzle.config.ts | 12 + .../drizzle/0000_nappy_james_howlett.sql | 46 +++ .../drizzle/meta/0000_snapshot.json | 304 ++++++++++++++++++ apps/chain-indexer/drizzle/meta/_journal.json | 13 + apps/chain-indexer/env/.env.sample | 11 + apps/chain-indexer/eslint.config.mjs | 11 + apps/chain-indexer/package.json | 60 ++++ apps/chain-indexer/src/app.ts | 20 ++ apps/chain-indexer/src/config/env.config.ts | 23 ++ apps/chain-indexer/src/db/bytea.ts | 7 + .../chain-indexer/src/db/pg-client.service.ts | 18 ++ apps/chain-indexer/src/db/schema.ts | 62 ++++ .../src/http-schemas/healthz.schema.ts | 9 + .../src/http-schemas/status.schema.ts | 17 + apps/chain-indexer/src/index.ts | 35 ++ .../src/lib/create-route/create-route.ts | 10 + .../src/pipeline/block-committer.service.ts | 90 ++++++ .../pipeline/block-decoder.service.spec.ts | 117 +++++++ .../src/pipeline/block-decoder.service.ts | 68 ++++ .../src/pipeline/canonical-json.spec.ts | 25 ++ .../src/pipeline/canonical-json.ts | 20 ++ .../src/pipeline/decoded-block.ts | 26 ++ .../src/pipeline/sync-runner.service.ts | 160 +++++++++ .../src/providers/app-config.provider.ts | 14 + .../src/providers/db.provider.ts | 30 ++ apps/chain-indexer/src/providers/index.ts | 5 + .../src/providers/logging.provider.ts | 13 + .../src/providers/raw-app-config.provider.ts | 8 + .../src/providers/type-registry.provider.ts | 36 +++ .../src/routes/healthz/healthz.router.ts | 26 ++ apps/chain-indexer/src/routes/index.ts | 2 + .../src/routes/status/status.router.ts | 29 ++ .../src/rpc/rpc-client-pool.service.spec.ts | 104 ++++++ .../src/rpc/rpc-client-pool.service.ts | 100 ++++++ apps/chain-indexer/src/rpc/rpc-types.ts | 36 +++ apps/chain-indexer/src/server.ts | 6 + .../services/app-config/app-config.service.ts | 12 + .../src/services/config/config.service.ts | 19 ++ .../hono-error-handler.service.ts | 102 ++++++ .../open-api-hono-handler.ts | 26 ++ .../shutdown-server/shutdown-server.ts | 31 ++ .../src/services/start-server/start-server.ts | 57 ++++ .../src/services/status/status.service.ts | 35 ++ apps/chain-indexer/src/types/app-context.ts | 7 + apps/chain-indexer/test/setup-unit-env.ts | 1 + apps/chain-indexer/test/setup-unit-tests.ts | 10 + apps/chain-indexer/tsconfig.build.json | 12 + apps/chain-indexer/tsconfig.json | 13 + apps/chain-indexer/tsup.config.ts | 24 ++ apps/chain-indexer/vitest.config.ts | 34 ++ package-lock.json | 79 +++++ 52 files changed, 2085 insertions(+) create mode 100644 apps/chain-indexer/README.md create mode 100644 apps/chain-indexer/drizzle.config.ts create mode 100644 apps/chain-indexer/drizzle/0000_nappy_james_howlett.sql create mode 100644 apps/chain-indexer/drizzle/meta/0000_snapshot.json create mode 100644 apps/chain-indexer/drizzle/meta/_journal.json create mode 100644 apps/chain-indexer/env/.env.sample create mode 100644 apps/chain-indexer/eslint.config.mjs create mode 100644 apps/chain-indexer/package.json create mode 100644 apps/chain-indexer/src/app.ts create mode 100644 apps/chain-indexer/src/config/env.config.ts create mode 100644 apps/chain-indexer/src/db/bytea.ts create mode 100644 apps/chain-indexer/src/db/pg-client.service.ts create mode 100644 apps/chain-indexer/src/db/schema.ts create mode 100644 apps/chain-indexer/src/http-schemas/healthz.schema.ts create mode 100644 apps/chain-indexer/src/http-schemas/status.schema.ts create mode 100644 apps/chain-indexer/src/index.ts create mode 100644 apps/chain-indexer/src/lib/create-route/create-route.ts create mode 100644 apps/chain-indexer/src/pipeline/block-committer.service.ts create mode 100644 apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts create mode 100644 apps/chain-indexer/src/pipeline/block-decoder.service.ts create mode 100644 apps/chain-indexer/src/pipeline/canonical-json.spec.ts create mode 100644 apps/chain-indexer/src/pipeline/canonical-json.ts create mode 100644 apps/chain-indexer/src/pipeline/decoded-block.ts create mode 100644 apps/chain-indexer/src/pipeline/sync-runner.service.ts create mode 100644 apps/chain-indexer/src/providers/app-config.provider.ts create mode 100644 apps/chain-indexer/src/providers/db.provider.ts create mode 100644 apps/chain-indexer/src/providers/index.ts create mode 100644 apps/chain-indexer/src/providers/logging.provider.ts create mode 100644 apps/chain-indexer/src/providers/raw-app-config.provider.ts create mode 100644 apps/chain-indexer/src/providers/type-registry.provider.ts create mode 100644 apps/chain-indexer/src/routes/healthz/healthz.router.ts create mode 100644 apps/chain-indexer/src/routes/index.ts create mode 100644 apps/chain-indexer/src/routes/status/status.router.ts create mode 100644 apps/chain-indexer/src/rpc/rpc-client-pool.service.spec.ts create mode 100644 apps/chain-indexer/src/rpc/rpc-client-pool.service.ts create mode 100644 apps/chain-indexer/src/rpc/rpc-types.ts create mode 100644 apps/chain-indexer/src/server.ts create mode 100644 apps/chain-indexer/src/services/app-config/app-config.service.ts create mode 100644 apps/chain-indexer/src/services/config/config.service.ts create mode 100644 apps/chain-indexer/src/services/hono-error-handler/hono-error-handler.service.ts create mode 100644 apps/chain-indexer/src/services/open-api-hono-handler/open-api-hono-handler.ts create mode 100644 apps/chain-indexer/src/services/shutdown-server/shutdown-server.ts create mode 100644 apps/chain-indexer/src/services/start-server/start-server.ts create mode 100644 apps/chain-indexer/src/services/status/status.service.ts create mode 100644 apps/chain-indexer/src/types/app-context.ts create mode 100644 apps/chain-indexer/test/setup-unit-env.ts create mode 100644 apps/chain-indexer/test/setup-unit-tests.ts create mode 100644 apps/chain-indexer/tsconfig.build.json create mode 100644 apps/chain-indexer/tsconfig.json create mode 100644 apps/chain-indexer/tsup.config.ts create mode 100644 apps/chain-indexer/vitest.config.ts diff --git a/apps/chain-indexer/README.md b/apps/chain-indexer/README.md new file mode 100644 index 0000000000..a98f36f8f3 --- /dev/null +++ b/apps/chain-indexer/README.md @@ -0,0 +1,50 @@ +# Chain Indexer + +Rewrite of the Akash blockchain indexer as an encapsulated app with its own database and, eventually, its own public REST API. Design doc and discussion live in the Linear project (see CON-803). + +One codebase, several processes. The role is picked at runtime: + +``` +INDEXER_ROLE = sync | backfill | api | jobs +NETWORK = mainnet | sandbox | testnet +``` + +Currently implemented: `sync` (live tail with per-block atomic commits, advisory-lock leader election, parent-hash continuity check) and a minimal `api` (healthz + status). `backfill` and `jobs` exit with `ROLE_NOT_IMPLEMENTED`. + +## Running locally + +Create `env/.env` with at least: + +``` +NETWORK=sandbox +POSTGRES_DB_URI=postgres://user:password@localhost:5432/chain-indexer +``` + +RPC endpoints default to the network's public nodes from `@akashnetwork/net`; override with a comma-separated `RPC_NODE_ENDPOINTS`. With no checkpoint in the database, sync starts at the current chain tip (set `SYNC_START_HEIGHT` to start elsewhere). Migrations run automatically on boot. + +```bash +npm run dev +``` + +Then check progress: + +```bash +curl localhost:3092/v1/status +``` + +The checkpoint height should advance as blocks land in `cosmos.blocks`, `cosmos.transactions`, and `cosmos.messages`. + +## Tests + +```bash +npm test +npm run lint -- --quiet +``` + +## Schema changes + +Edit `src/db/schema.ts`, then: + +```bash +npm run migration:gen +``` diff --git a/apps/chain-indexer/drizzle.config.ts b/apps/chain-indexer/drizzle.config.ts new file mode 100644 index 0000000000..cbfbbd5d94 --- /dev/null +++ b/apps/chain-indexer/drizzle.config.ts @@ -0,0 +1,12 @@ +import "@akashnetwork/env-loader"; + +import { defineConfig } from "drizzle-kit"; + +export default defineConfig({ + schema: "./src/db/schema.ts", + out: "./drizzle", + dialect: "postgresql", + dbCredentials: { + url: process.env.POSTGRES_DB_URI ?? "postgres://offline:offline@localhost:5432/offline" + } +}); diff --git a/apps/chain-indexer/drizzle/0000_nappy_james_howlett.sql b/apps/chain-indexer/drizzle/0000_nappy_james_howlett.sql new file mode 100644 index 0000000000..45923803e8 --- /dev/null +++ b/apps/chain-indexer/drizzle/0000_nappy_james_howlett.sql @@ -0,0 +1,46 @@ +CREATE SCHEMA "cosmos"; +--> statement-breakpoint +CREATE TABLE "cosmos"."blocks" ( + "height" bigint PRIMARY KEY NOT NULL, + "datetime" timestamp with time zone NOT NULL, + "hash" "bytea" NOT NULL, + "parent_hash" "bytea", + "proposer_address" text NOT NULL, + "tx_count" integer NOT NULL +); +--> statement-breakpoint +CREATE TABLE "indexer_state" ( + "stream" text PRIMARY KEY NOT NULL, + "last_height" bigint NOT NULL, + "updated_at" timestamp with time zone NOT NULL +); +--> statement-breakpoint +CREATE TABLE "cosmos"."message_types" ( + "id" "smallserial" PRIMARY KEY NOT NULL, + "type" text NOT NULL +); +--> statement-breakpoint +CREATE TABLE "cosmos"."messages" ( + "height" bigint NOT NULL, + "tx_index" integer NOT NULL, + "index" integer NOT NULL, + "type_id" smallint NOT NULL, + "body" jsonb, + CONSTRAINT "messages_height_tx_index_index_pk" PRIMARY KEY("height","tx_index","index") +); +--> statement-breakpoint +CREATE TABLE "cosmos"."transactions" ( + "height" bigint NOT NULL, + "index" integer NOT NULL, + "hash" "bytea" NOT NULL, + "code" integer NOT NULL, + "gas_used" bigint NOT NULL, + "gas_wanted" bigint NOT NULL, + "fee" jsonb NOT NULL, + CONSTRAINT "transactions_height_index_pk" PRIMARY KEY("height","index") +); +--> statement-breakpoint +ALTER TABLE "cosmos"."messages" ADD CONSTRAINT "messages_type_id_message_types_id_fk" FOREIGN KEY ("type_id") REFERENCES "cosmos"."message_types"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +CREATE UNIQUE INDEX "message_types_type_idx" ON "cosmos"."message_types" USING btree ("type");--> statement-breakpoint +CREATE INDEX "messages_type_id_idx" ON "cosmos"."messages" USING btree ("type_id");--> statement-breakpoint +CREATE INDEX "transactions_hash_idx" ON "cosmos"."transactions" USING btree ("hash"); \ No newline at end of file diff --git a/apps/chain-indexer/drizzle/meta/0000_snapshot.json b/apps/chain-indexer/drizzle/meta/0000_snapshot.json new file mode 100644 index 0000000000..cd0c1d0953 --- /dev/null +++ b/apps/chain-indexer/drizzle/meta/0000_snapshot.json @@ -0,0 +1,304 @@ +{ + "id": "6cc7b5d5-bf47-4e13-a098-4819d0a2863e", + "prevId": "00000000-0000-0000-0000-000000000000", + "version": "7", + "dialect": "postgresql", + "tables": { + "cosmos.blocks": { + "name": "blocks", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "datetime": { + "name": "datetime", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "parent_hash": { + "name": "parent_hash", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "proposer_address": { + "name": "proposer_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tx_count": { + "name": "tx_count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.indexer_state": { + "name": "indexer_state", + "schema": "", + "columns": { + "stream": { + "name": "stream", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "last_height": { + "name": "last_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.message_types": { + "name": "message_types", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "smallserial", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "message_types_type_idx": { + "name": "message_types_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.messages": { + "name": "messages", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type_id": { + "name": "type_id", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "body": { + "name": "body", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "messages_type_id_idx": { + "name": "messages_type_id_idx", + "columns": [ + { + "expression": "type_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "messages_type_id_message_types_id_fk": { + "name": "messages_type_id_message_types_id_fk", + "tableFrom": "messages", + "tableTo": "message_types", + "schemaTo": "cosmos", + "columnsFrom": [ + "type_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messages_height_tx_index_index_pk": { + "name": "messages_height_tx_index_index_pk", + "columns": [ + "height", + "tx_index", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.transactions": { + "name": "transactions", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "code": { + "name": "code", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "gas_used": { + "name": "gas_used", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gas_wanted": { + "name": "gas_wanted", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "fee": { + "name": "fee", + "type": "jsonb", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "transactions_hash_idx": { + "name": "transactions_hash_idx", + "columns": [ + { + "expression": "hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "transactions_height_index_pk": { + "name": "transactions_height_index_pk", + "columns": [ + "height", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": { + "cosmos": "cosmos" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/chain-indexer/drizzle/meta/_journal.json b/apps/chain-indexer/drizzle/meta/_journal.json new file mode 100644 index 0000000000..60910f3091 --- /dev/null +++ b/apps/chain-indexer/drizzle/meta/_journal.json @@ -0,0 +1,13 @@ +{ + "version": "7", + "dialect": "postgresql", + "entries": [ + { + "idx": 0, + "version": "7", + "when": 1786433105904, + "tag": "0000_nappy_james_howlett", + "breakpoints": true + } + ] +} \ No newline at end of file diff --git a/apps/chain-indexer/env/.env.sample b/apps/chain-indexer/env/.env.sample new file mode 100644 index 0000000000..ec4a54572a --- /dev/null +++ b/apps/chain-indexer/env/.env.sample @@ -0,0 +1,11 @@ +INDEXER_ROLE=sync +NETWORK=sandbox +POSTGRES_DB_URI=postgres://user:password@localhost:5432/chain-indexer +RPC_NODE_ENDPOINTS= +SYNC_START_HEIGHT= +SYNC_POLL_INTERVAL_MS=3000 +RPC_TIMEOUT_MS=15000 +MESSAGE_BODY_MAX_BYTES=65536 +PORT=3092 +LOG_LEVEL=info +STD_OUT_LOG_FORMAT=pretty diff --git a/apps/chain-indexer/eslint.config.mjs b/apps/chain-indexer/eslint.config.mjs new file mode 100644 index 0000000000..5f06f758f0 --- /dev/null +++ b/apps/chain-indexer/eslint.config.mjs @@ -0,0 +1,11 @@ +import tsConfig from "@akashnetwork/dev-config/eslint/typescript.mjs"; + +export default [ + ...tsConfig, + { + files: ["**/*.ts", "**/*.tsx"], + rules: { + "@typescript-eslint/explicit-module-boundary-types": ["error"] + } + } +]; diff --git a/apps/chain-indexer/package.json b/apps/chain-indexer/package.json new file mode 100644 index 0000000000..f340e9a936 --- /dev/null +++ b/apps/chain-indexer/package.json @@ -0,0 +1,60 @@ +{ + "name": "@akashnetwork/chain-indexer", + "version": "0.0.1", + "description": "Akash blockchain indexer: sync, backfill, api, and jobs roles over a dedicated database", + "license": "Apache-2.0", + "author": "Akash Network", + "main": "dist/server.js", + "scripts": { + "build": "NODE_ENV=production tsup", + "dev": "npm run start", + "dev-nodc": "npm run start", + "format": "prettier --write ./*.{ts,js,json} **/*.{ts,js,json}", + "lint": "eslint .", + "migration:gen": "drizzle-kit generate", + "prod": "node --enable-source-maps --require ./dist/instrumentation.js ./dist/server.js", + "start": "tsup --watch", + "test": "vitest run --project unit", + "test:cov": "vitest run --project unit --coverage", + "test:unit": "vitest run --project unit" + }, + "dependencies": { + "@akashnetwork/chain-sdk": "1.0.0-alpha.41", + "@akashnetwork/env-loader": "*", + "@akashnetwork/instrumentation": "*", + "@akashnetwork/logging": "*", + "@akashnetwork/net": "*", + "@cosmjs/encoding": "~0.38.0", + "@cosmjs/proto-signing": "~0.38.0", + "@cosmjs/stargate": "~0.38.0", + "@hono/node-server": "1.13.7", + "@hono/otel": "~0.4.0", + "@hono/zod-openapi": "0.18.4", + "@opentelemetry/api": "^1.9.0", + "drizzle-orm": "^0.45.2", + "hono": "4.6.12", + "http-errors": "^2.0.0", + "lodash": "^4.17.21", + "postgres": "^3.4.4", + "reflect-metadata": "^0.2.2", + "tsyringe": "^4.10.0", + "zod": "3.*" + }, + "devDependencies": { + "@akashnetwork/dev-config": "*", + "@types/lodash": "^4.17.0", + "@types/node": "^22.13.11", + "@typescript-eslint/eslint-plugin": "^8.64.0", + "@vitest/coverage-v8": "^4.1.5", + "cosmjs-types": "~0.11.0", + "drizzle-kit": "^0.31.10", + "eslint": "^9.39.5", + "eslint-config-next": "^15.5.20", + "eslint-plugin-simple-import-sort": "^13.0.0", + "prettier": "^3.3.0", + "tsup": "^8.5.1", + "typescript": "~5.8.2", + "vitest": "^4.1.5", + "vitest-mock-extended": "^4.0.0" + } +} diff --git a/apps/chain-indexer/src/app.ts b/apps/chain-indexer/src/app.ts new file mode 100644 index 0000000000..473f2f144f --- /dev/null +++ b/apps/chain-indexer/src/app.ts @@ -0,0 +1,20 @@ +import { HttpLoggerInterceptor } from "@akashnetwork/logging/hono"; +import { otel } from "@hono/otel"; +import { Hono } from "hono"; +import { container } from "tsyringe"; + +import { healthzRouter, statusRouter } from "@src/routes"; +import { HonoErrorHandlerService } from "@src/services/hono-error-handler/hono-error-handler.service"; +import type { AppEnv } from "@src/types/app-context"; + +export function createApp(): Hono { + const app = new Hono(); + + app.use("*", otel({ captureRequestHeaders: ["baggage"] })); + app.use(container.resolve(HttpLoggerInterceptor).intercept()); + app.route("/", healthzRouter); + app.route("/", statusRouter); + app.onError(container.resolve(HonoErrorHandlerService).handle); + + return app; +} diff --git a/apps/chain-indexer/src/config/env.config.ts b/apps/chain-indexer/src/config/env.config.ts new file mode 100644 index 0000000000..25982e1e9c --- /dev/null +++ b/apps/chain-indexer/src/config/env.config.ts @@ -0,0 +1,23 @@ +import { z } from "zod"; + +export const envSchema = z.object({ + INDEXER_ROLE: z.enum(["sync", "backfill", "api", "jobs"]).default("sync"), + NETWORK: z.enum(["mainnet", "sandbox", "testnet"]).default("sandbox"), + POSTGRES_DB_URI: z.string(), + /** Comma-separated RPC endpoints. Defaults to the network's public endpoints from @akashnetwork/net. */ + RPC_NODE_ENDPOINTS: z.string().optional(), + RPC_TIMEOUT_MS: z.number({ coerce: true }).int().positive().default(15_000), + RPC_NODE_COOLDOWN_MS: z.number({ coerce: true }).int().positive().default(30_000), + /** First height to sync when the database has no checkpoint yet. Defaults to the current chain tip. */ + SYNC_START_HEIGHT: z.number({ coerce: true }).int().positive().optional(), + SYNC_POLL_INTERVAL_MS: z.number({ coerce: true }).int().positive().default(3_000), + /** Decoded message bodies above this serialized size are stored as null to keep pathological messages out of Postgres. */ + MESSAGE_BODY_MAX_BYTES: z.number({ coerce: true }).int().positive().default(65_536), + DRIZZLE_MIGRATIONS_FOLDER: z.string().default("./drizzle"), + LOG_LEVEL: z.enum(["fatal", "error", "warn", "info", "debug", "trace"]).optional().default("info"), + STD_OUT_LOG_FORMAT: z.enum(["json", "pretty"]).optional().default("json"), + NODE_ENV: z.enum(["development", "production", "test"]).optional().default("development"), + PORT: z.number({ coerce: true }).optional().default(3092) +}); + +export type EnvConfig = z.infer; diff --git a/apps/chain-indexer/src/db/bytea.ts b/apps/chain-indexer/src/db/bytea.ts new file mode 100644 index 0000000000..7b936dbf33 --- /dev/null +++ b/apps/chain-indexer/src/db/bytea.ts @@ -0,0 +1,7 @@ +import { customType } from "drizzle-orm/pg-core"; + +export const bytea = customType<{ data: Buffer; driverData: Buffer }>({ + dataType() { + return "bytea"; + } +}); diff --git a/apps/chain-indexer/src/db/pg-client.service.ts b/apps/chain-indexer/src/db/pg-client.service.ts new file mode 100644 index 0000000000..68c12b945a --- /dev/null +++ b/apps/chain-indexer/src/db/pg-client.service.ts @@ -0,0 +1,18 @@ +import postgres from "postgres"; +import { inject, singleton } from "tsyringe"; + +import type { EnvConfig } from "@src/config/env.config"; +import { APP_CONFIG } from "@src/providers/app-config.provider"; + +@singleton() +export class PgClientService { + readonly client: postgres.Sql; + + constructor(@inject(APP_CONFIG) config: EnvConfig) { + this.client = postgres(config.POSTGRES_DB_URI, { max: 10 }); + } + + async dispose(): Promise { + await this.client.end({ timeout: 5 }); + } +} diff --git a/apps/chain-indexer/src/db/schema.ts b/apps/chain-indexer/src/db/schema.ts new file mode 100644 index 0000000000..98c59a8dc6 --- /dev/null +++ b/apps/chain-indexer/src/db/schema.ts @@ -0,0 +1,62 @@ +import { bigint, index, integer, jsonb, pgSchema, pgTable, primaryKey, smallint, smallserial, text, timestamp, uniqueIndex } from "drizzle-orm/pg-core"; + +import { bytea } from "@src/db/bytea"; + +export const cosmosSchema = pgSchema("cosmos"); + +export const Blocks = cosmosSchema.table("blocks", { + height: bigint("height", { mode: "number" }).primaryKey(), + datetime: timestamp("datetime", { withTimezone: true }).notNull(), + hash: bytea("hash").notNull(), + parentHash: bytea("parent_hash"), + proposerAddress: text("proposer_address").notNull(), + txCount: integer("tx_count").notNull() +}); + +export interface FeeCoin { + denom: string; + amount: string; +} + +export const Transactions = cosmosSchema.table( + "transactions", + { + height: bigint("height", { mode: "number" }).notNull(), + index: integer("index").notNull(), + hash: bytea("hash").notNull(), + code: integer("code").notNull(), + gasUsed: bigint("gas_used", { mode: "number" }).notNull(), + gasWanted: bigint("gas_wanted", { mode: "number" }).notNull(), + fee: jsonb("fee").$type().notNull() + }, + t => [primaryKey({ columns: [t.height, t.index] }), index("transactions_hash_idx").on(t.hash)] +); + +export const MessageTypes = cosmosSchema.table( + "message_types", + { + id: smallserial("id").primaryKey(), + type: text("type").notNull() + }, + t => [uniqueIndex("message_types_type_idx").on(t.type)] +); + +export const Messages = cosmosSchema.table( + "messages", + { + height: bigint("height", { mode: "number" }).notNull(), + txIndex: integer("tx_index").notNull(), + index: integer("index").notNull(), + typeId: smallint("type_id") + .notNull() + .references(() => MessageTypes.id), + body: jsonb("body") + }, + t => [primaryKey({ columns: [t.height, t.txIndex, t.index] }), index("messages_type_id_idx").on(t.typeId)] +); + +export const IndexerState = pgTable("indexer_state", { + stream: text("stream").primaryKey(), + lastHeight: bigint("last_height", { mode: "number" }).notNull(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull() +}); diff --git a/apps/chain-indexer/src/http-schemas/healthz.schema.ts b/apps/chain-indexer/src/http-schemas/healthz.schema.ts new file mode 100644 index 0000000000..55c4801d7c --- /dev/null +++ b/apps/chain-indexer/src/http-schemas/healthz.schema.ts @@ -0,0 +1,9 @@ +import { z } from "zod"; + +export const HealthzResponseSchema = z.object({ + data: z.object({ + status: z.literal("ok") + }) +}); + +export type HealthzResponse = z.infer; diff --git a/apps/chain-indexer/src/http-schemas/status.schema.ts b/apps/chain-indexer/src/http-schemas/status.schema.ts new file mode 100644 index 0000000000..43c43b1a62 --- /dev/null +++ b/apps/chain-indexer/src/http-schemas/status.schema.ts @@ -0,0 +1,17 @@ +import { z } from "zod"; + +export const StatusResponseSchema = z.object({ + data: z.object({ + network: z.string(), + role: z.string(), + checkpoints: z.array( + z.object({ + stream: z.string(), + lastHeight: z.number(), + updatedAt: z.string() + }) + ) + }) +}); + +export type StatusResponse = z.infer; diff --git a/apps/chain-indexer/src/index.ts b/apps/chain-indexer/src/index.ts new file mode 100644 index 0000000000..14a6eab0a8 --- /dev/null +++ b/apps/chain-indexer/src/index.ts @@ -0,0 +1,35 @@ +import "reflect-metadata"; +import "@src/providers"; + +import { createOtelLogger } from "@akashnetwork/logging/otel"; +import { container } from "tsyringe"; + +import { createApp } from "@src/app"; +import { SyncRunnerService } from "@src/pipeline/sync-runner.service"; +import { migrateDb } from "@src/providers/db.provider"; +import { AppConfigService } from "@src/services/app-config/app-config.service"; +import { startServer } from "@src/services/start-server/start-server"; + +export async function bootstrap(): Promise { + const config = container.resolve(AppConfigService); + const role = config.get("INDEXER_ROLE"); + const logger = createOtelLogger({ context: "APP" }); + + switch (role) { + case "sync": { + await migrateDb(); + await startServer(createApp(), logger, process, { port: config.get("PORT") }); + await container.resolve(SyncRunnerService).start(); + return; + } + case "api": { + await migrateDb(); + await startServer(createApp(), logger, process, { port: config.get("PORT") }); + return; + } + default: { + logger.error({ event: "ROLE_NOT_IMPLEMENTED", role }); + process.exitCode = 1; + } + } +} diff --git a/apps/chain-indexer/src/lib/create-route/create-route.ts b/apps/chain-indexer/src/lib/create-route/create-route.ts new file mode 100644 index 0000000000..247034159b --- /dev/null +++ b/apps/chain-indexer/src/lib/create-route/create-route.ts @@ -0,0 +1,10 @@ +import type { RouteConfig } from "@hono/zod-openapi"; +import { createRoute as createOpenApiRoute } from "@hono/zod-openapi"; + +export function createRoute< + R extends Omit & { + security?: Required["security"]; + } +>(routeConfig: R): Omit & { getRoutingPath(): string } { + return createOpenApiRoute(routeConfig as Omit); +} diff --git a/apps/chain-indexer/src/pipeline/block-committer.service.ts b/apps/chain-indexer/src/pipeline/block-committer.service.ts new file mode 100644 index 0000000000..a730080b38 --- /dev/null +++ b/apps/chain-indexer/src/pipeline/block-committer.service.ts @@ -0,0 +1,90 @@ +import { inArray } from "drizzle-orm"; +import { inject, singleton } from "tsyringe"; + +import { Blocks, IndexerState, Messages, MessageTypes, Transactions } from "@src/db/schema"; +import type { DecodedBlock } from "@src/pipeline/decoded-block"; +import type { ChainDatabase } from "@src/providers/db.provider"; +import { CHAIN_DB } from "@src/providers/db.provider"; + +export const SYNC_STREAM = "sync"; + +@singleton() +export class BlockCommitterService { + readonly #db: ChainDatabase; + readonly #typeIds = new Map(); + + constructor(@inject(CHAIN_DB) db: ChainDatabase) { + this.#db = db; + } + + async commit(block: DecodedBlock): Promise { + const typeIds = await this.#internMessageTypes(block); + + const transactionRows = block.transactions.map(tx => ({ + height: block.height, + index: tx.index, + hash: tx.hash, + code: tx.code, + gasUsed: tx.gasUsed, + gasWanted: tx.gasWanted, + fee: tx.fee + })); + + const messageRows = block.transactions.flatMap(tx => + tx.messages.map(message => ({ + height: block.height, + txIndex: tx.index, + index: message.index, + typeId: typeIds.get(message.typeUrl) as number, + body: message.body + })) + ); + + await this.#db.transaction(async tx => { + await tx + .insert(Blocks) + .values({ + height: block.height, + datetime: block.datetime, + hash: block.hash, + parentHash: block.parentHash, + proposerAddress: block.proposerAddress, + txCount: block.transactions.length + }) + .onConflictDoNothing(); + + if (transactionRows.length > 0) { + await tx.insert(Transactions).values(transactionRows).onConflictDoNothing(); + } + + if (messageRows.length > 0) { + await tx.insert(Messages).values(messageRows).onConflictDoNothing(); + } + + await tx + .insert(IndexerState) + .values({ stream: SYNC_STREAM, lastHeight: block.height, updatedAt: new Date() }) + .onConflictDoUpdate({ + target: IndexerState.stream, + set: { lastHeight: block.height, updatedAt: new Date() } + }); + }); + } + + async #internMessageTypes(block: DecodedBlock): Promise> { + const typeUrls = new Set(block.transactions.flatMap(tx => tx.messages.map(message => message.typeUrl))); + const missing = [...typeUrls].filter(typeUrl => !this.#typeIds.has(typeUrl)); + + if (missing.length > 0) { + await this.#db + .insert(MessageTypes) + .values(missing.map(type => ({ type }))) + .onConflictDoNothing(); + + const rows = await this.#db.select().from(MessageTypes).where(inArray(MessageTypes.type, missing)); + rows.forEach(row => this.#typeIds.set(row.type, row.id)); + } + + return this.#typeIds; + } +} diff --git a/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts b/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts new file mode 100644 index 0000000000..ed144eea6b --- /dev/null +++ b/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts @@ -0,0 +1,117 @@ +import { Registry } from "@cosmjs/proto-signing"; +import { defaultRegistryTypes } from "@cosmjs/stargate"; +import { MsgSend } from "cosmjs-types/cosmos/bank/v1beta1/tx"; +import { AuthInfo, TxBody, TxRaw } from "cosmjs-types/cosmos/tx/v1beta1/tx"; +import { createHash } from "node:crypto"; +import { describe, expect, it } from "vitest"; + +import { envSchema } from "@src/config/env.config"; +import { BlockDecoderService } from "@src/pipeline/block-decoder.service"; +import type { RpcBlockResult, RpcBlockResultsResult } from "@src/rpc/rpc-types"; + +describe(BlockDecoderService.name, () => { + it("decodes block metadata with hashes as buffers", () => { + const { decoder } = setup(); + + const decoded = decoder.decode(buildBlock({ txs: [] }), buildBlockResults([])); + + expect(decoded.height).toBe(1234); + expect(decoded.datetime).toEqual(new Date("2026-08-11T00:00:00Z")); + expect(decoded.hash).toEqual(Buffer.from("aa".repeat(32), "hex")); + expect(decoded.parentHash).toEqual(Buffer.from("bb".repeat(32), "hex")); + expect(decoded.proposerAddress).toBe("PROPOSER"); + expect(decoded.transactions).toEqual([]); + }); + + it("decodes a transaction with its sha256 hash, gas, fee, and typed message body", () => { + const { decoder } = setup(); + const rawTx = buildMsgSendTx(); + + const decoded = decoder.decode(buildBlock({ txs: [rawTx.toString("base64")] }), buildBlockResults([{ code: 0, gas_used: "51000", gas_wanted: "70000" }])); + + const [tx] = decoded.transactions; + expect(tx.hash).toEqual(createHash("sha256").update(rawTx).digest()); + expect(tx.code).toBe(0); + expect(tx.gasUsed).toBe(51000); + expect(tx.gasWanted).toBe(70000); + expect(tx.fee).toEqual([{ denom: "uakt", amount: "5000" }]); + expect(tx.messages).toHaveLength(1); + expect(tx.messages[0].typeUrl).toBe("/cosmos.bank.v1beta1.MsgSend"); + expect(tx.messages[0].body).toMatchObject({ + fromAddress: "akash1from", + toAddress: "akash1to", + amount: [{ denom: "uakt", amount: "42" }] + }); + }); + + it("stores a null body for message types missing from the registry", () => { + const { decoder } = setup(); + const rawTx = buildMsgSendTx("/akash.unknown.v1.MsgMystery"); + + const decoded = decoder.decode(buildBlock({ txs: [rawTx.toString("base64")] }), buildBlockResults([{ code: 0, gas_used: "0", gas_wanted: "0" }])); + + expect(decoded.transactions[0].messages[0].typeUrl).toBe("/akash.unknown.v1.MsgMystery"); + expect(decoded.transactions[0].messages[0].body).toBeNull(); + }); + + it("stores a null body when the decoded message exceeds the size cap", () => { + const { decoder } = setup({ maxBodyBytes: 10 }); + const rawTx = buildMsgSendTx(); + + const decoded = decoder.decode(buildBlock({ txs: [rawTx.toString("base64")] }), buildBlockResults([{ code: 0, gas_used: "0", gas_wanted: "0" }])); + + expect(decoded.transactions[0].messages[0].body).toBeNull(); + }); + + it("marks failed transactions with their error code", () => { + const { decoder } = setup(); + const rawTx = buildMsgSendTx(); + + const decoded = decoder.decode(buildBlock({ txs: [rawTx.toString("base64")] }), buildBlockResults([{ code: 11, gas_used: "70000", gas_wanted: "70000" }])); + + expect(decoded.transactions[0].code).toBe(11); + }); + + function setup(input?: { maxBodyBytes?: number }) { + const config = envSchema.parse({ + POSTGRES_DB_URI: "postgres://unit:unit@localhost:5432/unit", + ...(input?.maxBodyBytes ? { MESSAGE_BODY_MAX_BYTES: input.maxBodyBytes } : {}) + }); + const decoder = new BlockDecoderService(new Registry(defaultRegistryTypes), config); + return { decoder }; + } + + function buildBlock(input: { txs: string[] }): RpcBlockResult { + return { + block_id: { hash: "AA".repeat(32) }, + block: { + header: { + height: "1234", + time: "2026-08-11T00:00:00Z", + proposer_address: "PROPOSER", + last_block_id: { hash: "BB".repeat(32) } + }, + data: { txs: input.txs } + } + }; + } + + function buildBlockResults(txsResults: { code: number; gas_used: string; gas_wanted: string }[]): RpcBlockResultsResult { + return { height: "1234", txs_results: txsResults }; + } + + function buildMsgSendTx(typeUrl = "/cosmos.bank.v1beta1.MsgSend"): Buffer { + const message = MsgSend.encode({ + fromAddress: "akash1from", + toAddress: "akash1to", + amount: [{ denom: "uakt", amount: "42" }] + }).finish(); + + const bodyBytes = TxBody.encode(TxBody.fromPartial({ messages: [{ typeUrl, value: message }] })).finish(); + const authInfoBytes = AuthInfo.encode( + AuthInfo.fromPartial({ fee: { amount: [{ denom: "uakt", amount: "5000" }], gasLimit: 70_000n, payer: "", granter: "" }, signerInfos: [] }) + ).finish(); + + return Buffer.from(TxRaw.encode(TxRaw.fromPartial({ bodyBytes, authInfoBytes, signatures: [new Uint8Array(64)] })).finish()); + } +}); diff --git a/apps/chain-indexer/src/pipeline/block-decoder.service.ts b/apps/chain-indexer/src/pipeline/block-decoder.service.ts new file mode 100644 index 0000000000..01e7b0b258 --- /dev/null +++ b/apps/chain-indexer/src/pipeline/block-decoder.service.ts @@ -0,0 +1,68 @@ +import { fromBase64 } from "@cosmjs/encoding"; +import { decodeTxRaw } from "@cosmjs/proto-signing"; +import { createHash } from "node:crypto"; +import { inject, singleton } from "tsyringe"; + +import type { EnvConfig } from "@src/config/env.config"; +import { toCanonicalJson } from "@src/pipeline/canonical-json"; +import type { DecodedBlock, DecodedMessage, DecodedTransaction } from "@src/pipeline/decoded-block"; +import { APP_CONFIG } from "@src/providers/app-config.provider"; +import type { Registry } from "@src/providers/type-registry.provider"; +import { TYPE_REGISTRY } from "@src/providers/type-registry.provider"; +import type { RpcBlockResult, RpcBlockResultsResult, RpcTxResult } from "@src/rpc/rpc-types"; + +@singleton() +export class BlockDecoderService { + readonly #registry: Registry; + readonly #maxBodyBytes: number; + + constructor(@inject(TYPE_REGISTRY) registry: Registry, @inject(APP_CONFIG) config: EnvConfig) { + this.#registry = registry; + this.#maxBodyBytes = config.MESSAGE_BODY_MAX_BYTES; + } + + decode(block: RpcBlockResult, blockResults: RpcBlockResultsResult): DecodedBlock { + const rawTxs = block.block.data.txs; + const txResults = blockResults.txs_results ?? []; + + return { + height: parseInt(block.block.header.height), + datetime: new Date(block.block.header.time), + hash: Buffer.from(block.block_id.hash, "hex"), + parentHash: block.block.header.last_block_id?.hash ? Buffer.from(block.block.header.last_block_id.hash, "hex") : null, + proposerAddress: block.block.header.proposer_address, + transactions: rawTxs.map((rawTx, index) => this.#decodeTransaction(rawTx, txResults[index], index)) + }; + } + + #decodeTransaction(rawTxBase64: string, txResult: RpcTxResult | undefined, index: number): DecodedTransaction { + const rawTx = fromBase64(rawTxBase64); + const decodedTx = decodeTxRaw(rawTx); + + return { + index, + hash: createHash("sha256").update(rawTx).digest(), + code: txResult?.code ?? 0, + gasUsed: parseInt(txResult?.gas_used ?? "0"), + gasWanted: parseInt(txResult?.gas_wanted ?? "0"), + fee: decodedTx.authInfo.fee?.amount.map(({ denom, amount }) => ({ denom, amount })) ?? [], + messages: decodedTx.body.messages.map((message, messageIndex) => this.#decodeMessage(message, messageIndex)) + }; + } + + #decodeMessage(message: { typeUrl: string; value: Uint8Array }, index: number): DecodedMessage { + return { + index, + typeUrl: message.typeUrl, + body: this.#decodeBody(message) + }; + } + + #decodeBody(message: { typeUrl: string; value: Uint8Array }): unknown | null { + try { + return toCanonicalJson(this.#registry.decode(message), this.#maxBodyBytes); + } catch { + return null; + } + } +} diff --git a/apps/chain-indexer/src/pipeline/canonical-json.spec.ts b/apps/chain-indexer/src/pipeline/canonical-json.spec.ts new file mode 100644 index 0000000000..f9d7476ea2 --- /dev/null +++ b/apps/chain-indexer/src/pipeline/canonical-json.spec.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; + +import { toCanonicalJson } from "@src/pipeline/canonical-json"; + +describe(toCanonicalJson.name, () => { + it("serializes plain objects unchanged", () => { + expect(toCanonicalJson({ denom: "uakt", amount: "100" }, 1_000)).toEqual({ denom: "uakt", amount: "100" }); + }); + + it("converts bigint values to strings", () => { + expect(toCanonicalJson({ gasLimit: 200_000n }, 1_000)).toEqual({ gasLimit: "200000" }); + }); + + it("converts byte arrays to base64 strings", () => { + expect(toCanonicalJson({ payload: Uint8Array.from([1, 2, 3]) }, 1_000)).toEqual({ payload: Buffer.from([1, 2, 3]).toString("base64") }); + }); + + it("returns null when the serialized value exceeds maxBytes", () => { + expect(toCanonicalJson({ memo: "x".repeat(100) }, 50)).toBeNull(); + }); + + it("returns null for undefined values", () => { + expect(toCanonicalJson(undefined, 1_000)).toBeNull(); + }); +}); diff --git a/apps/chain-indexer/src/pipeline/canonical-json.ts b/apps/chain-indexer/src/pipeline/canonical-json.ts new file mode 100644 index 0000000000..cce85c9f01 --- /dev/null +++ b/apps/chain-indexer/src/pipeline/canonical-json.ts @@ -0,0 +1,20 @@ +function replacer(this: unknown, key: string, value: unknown): unknown { + if (typeof value === "bigint") { + return value.toString(); + } + if (value instanceof Uint8Array) { + return Buffer.from(value).toString("base64"); + } + return value; +} + +/** Serializes a decoded proto message into plain JSON, returning null when it exceeds maxBytes. */ +export function toCanonicalJson(value: unknown, maxBytes: number): unknown | null { + const json = JSON.stringify(value, replacer); + + if (json === undefined || Buffer.byteLength(json) > maxBytes) { + return null; + } + + return JSON.parse(json); +} diff --git a/apps/chain-indexer/src/pipeline/decoded-block.ts b/apps/chain-indexer/src/pipeline/decoded-block.ts new file mode 100644 index 0000000000..c1225790a0 --- /dev/null +++ b/apps/chain-indexer/src/pipeline/decoded-block.ts @@ -0,0 +1,26 @@ +import type { FeeCoin } from "@src/db/schema"; + +export interface DecodedMessage { + index: number; + typeUrl: string; + body: unknown | null; +} + +export interface DecodedTransaction { + index: number; + hash: Buffer; + code: number; + gasUsed: number; + gasWanted: number; + fee: FeeCoin[]; + messages: DecodedMessage[]; +} + +export interface DecodedBlock { + height: number; + datetime: Date; + hash: Buffer; + parentHash: Buffer | null; + proposerAddress: string; + transactions: DecodedTransaction[]; +} diff --git a/apps/chain-indexer/src/pipeline/sync-runner.service.ts b/apps/chain-indexer/src/pipeline/sync-runner.service.ts new file mode 100644 index 0000000000..acdbc27074 --- /dev/null +++ b/apps/chain-indexer/src/pipeline/sync-runner.service.ts @@ -0,0 +1,160 @@ +import { eq } from "drizzle-orm"; +import { setTimeout as delay } from "node:timers/promises"; +import type postgres from "postgres"; +import { inject, singleton } from "tsyringe"; + +import type { EnvConfig } from "@src/config/env.config"; +import { PgClientService } from "@src/db/pg-client.service"; +import { Blocks, IndexerState } from "@src/db/schema"; +import { BlockCommitterService, SYNC_STREAM } from "@src/pipeline/block-committer.service"; +import { BlockDecoderService } from "@src/pipeline/block-decoder.service"; +import type { DecodedBlock } from "@src/pipeline/decoded-block"; +import { APP_CONFIG } from "@src/providers/app-config.provider"; +import type { ChainDatabase } from "@src/providers/db.provider"; +import { CHAIN_DB } from "@src/providers/db.provider"; +import { LoggerService } from "@src/providers/logging.provider"; +import { RpcClientPool } from "@src/rpc/rpc-client-pool.service"; + +/** Arbitrary but fixed application-wide key for the sync leader pg advisory lock. */ +const SYNC_LEADER_LOCK_KEY = 7_431_001; +const LEADERSHIP_RETRY_MS = 5_000; +const PROGRESS_LOG_EVERY_BLOCKS = 100; + +@singleton() +export class SyncRunnerService { + readonly #pgClient: PgClientService; + readonly #db: ChainDatabase; + readonly #pool: RpcClientPool; + readonly #decoder: BlockDecoderService; + readonly #committer: BlockCommitterService; + readonly #config: EnvConfig; + readonly #logger: LoggerService; + + #stopped = false; + #reserved: postgres.ReservedSql | undefined; + #lastHash: Buffer | null = null; + + constructor( + @inject(PgClientService) pgClient: PgClientService, + @inject(CHAIN_DB) db: ChainDatabase, + @inject(RpcClientPool) pool: RpcClientPool, + @inject(BlockDecoderService) decoder: BlockDecoderService, + @inject(BlockCommitterService) committer: BlockCommitterService, + @inject(APP_CONFIG) config: EnvConfig, + @inject(LoggerService) logger: LoggerService + ) { + this.#pgClient = pgClient; + this.#db = db; + this.#pool = pool; + this.#decoder = decoder; + this.#committer = committer; + this.#config = config; + this.#logger = logger; + this.#logger.setContext("SYNC"); + } + + async start(): Promise { + try { + await this.#run(); + } catch (error) { + if (this.#stopped) { + this.#logger.info({ event: "SYNC_STOPPED_DURING_SHUTDOWN" }); + return; + } + throw error; + } + } + + async dispose(): Promise { + this.#stopped = true; + try { + this.#reserved?.release(); + } catch { + /** the reserved connection may already be gone when the pool was ended first */ + } + } + + async #run(): Promise { + await this.#acquireLeadership(); + let nextHeight = await this.#resolveStartHeight(); + this.#logger.info({ event: "SYNC_STARTED", network: this.#config.NETWORK, nextHeight }); + + while (!this.#stopped) { + const tipHeight = await this.#getTipHeight(); + + if (nextHeight > tipHeight) { + await delay(this.#config.SYNC_POLL_INTERVAL_MS); + continue; + } + + while (nextHeight <= tipHeight && !this.#stopped) { + await this.#syncBlock(nextHeight); + nextHeight++; + } + } + } + + async #syncBlock(height: number): Promise { + const [block, blockResults] = await Promise.all([this.#pool.getBlock(height), this.#pool.getBlockResults(height)]); + const decoded = this.#decoder.decode(block, blockResults); + + this.#verifyContinuity(decoded); + await this.#committer.commit(decoded); + this.#lastHash = decoded.hash; + + if (height % PROGRESS_LOG_EVERY_BLOCKS === 0) { + this.#logger.info({ event: "SYNC_PROGRESS", height, txCount: decoded.transactions.length }); + } else { + this.#logger.debug({ event: "BLOCK_COMMITTED", height, txCount: decoded.transactions.length }); + } + } + + #verifyContinuity(block: DecodedBlock): void { + if (this.#lastHash && block.parentHash && !block.parentHash.equals(this.#lastHash)) { + this.#logger.error({ + event: "CHAIN_CONTINUITY_BROKEN", + height: block.height, + expectedParentHash: this.#lastHash.toString("hex"), + actualParentHash: block.parentHash.toString("hex") + }); + throw new Error(`Parent hash mismatch at height ${block.height}; halting sync`); + } + } + + async #acquireLeadership(): Promise { + this.#reserved = await this.#pgClient.client.reserve(); + + while (!this.#stopped) { + const [{ acquired }] = await this.#reserved`SELECT pg_try_advisory_lock(${SYNC_LEADER_LOCK_KEY}) AS acquired`; + + if (acquired) { + this.#logger.info({ event: "SYNC_LEADERSHIP_ACQUIRED" }); + return; + } + + this.#logger.info({ event: "SYNC_LEADERSHIP_WAITING" }); + await delay(LEADERSHIP_RETRY_MS); + } + } + + async #resolveStartHeight(): Promise { + const [state] = await this.#db.select().from(IndexerState).where(eq(IndexerState.stream, SYNC_STREAM)); + + if (state) { + const [checkpointBlock] = await this.#db.select().from(Blocks).where(eq(Blocks.height, state.lastHeight)); + this.#lastHash = checkpointBlock?.hash ?? null; + return state.lastHeight + 1; + } + + if (this.#config.SYNC_START_HEIGHT) { + return this.#config.SYNC_START_HEIGHT; + } + + return await this.#getTipHeight(); + } + + async #getTipHeight(): Promise { + const status = await this.#pool.getStatus(); + return parseInt(status.sync_info.latest_block_height); + } +} diff --git a/apps/chain-indexer/src/providers/app-config.provider.ts b/apps/chain-indexer/src/providers/app-config.provider.ts new file mode 100644 index 0000000000..6936da490b --- /dev/null +++ b/apps/chain-indexer/src/providers/app-config.provider.ts @@ -0,0 +1,14 @@ +import type { InjectionToken } from "tsyringe"; +import { container, instancePerContainerCachingFactory } from "tsyringe"; + +import type { EnvConfig } from "@src/config/env.config"; +import { envSchema } from "@src/config/env.config"; +import { RAW_APP_CONFIG } from "@src/providers/raw-app-config.provider"; + +export const APP_CONFIG: InjectionToken = Symbol("APP_CONFIG"); + +container.register(APP_CONFIG, { + useFactory: instancePerContainerCachingFactory(c => envSchema.parse(c.resolve(RAW_APP_CONFIG))) +}); + +export type { EnvConfig }; diff --git a/apps/chain-indexer/src/providers/db.provider.ts b/apps/chain-indexer/src/providers/db.provider.ts new file mode 100644 index 0000000000..90e837c5a7 --- /dev/null +++ b/apps/chain-indexer/src/providers/db.provider.ts @@ -0,0 +1,30 @@ +import { drizzle } from "drizzle-orm/postgres-js"; +import { migrate } from "drizzle-orm/postgres-js/migrator"; +import postgres from "postgres"; +import type { DependencyContainer, InjectionToken } from "tsyringe"; +import { container, instancePerContainerCachingFactory } from "tsyringe"; + +import { PgClientService } from "@src/db/pg-client.service"; +import * as schema from "@src/db/schema"; +import { APP_CONFIG } from "@src/providers/app-config.provider"; + +const createDatabase = (c: DependencyContainer) => drizzle(c.resolve(PgClientService).client, { schema }); + +export type ChainDatabase = ReturnType; +export const CHAIN_DB: InjectionToken = Symbol("CHAIN_DB"); + +container.register(CHAIN_DB, { + useFactory: instancePerContainerCachingFactory(createDatabase) +}); + +export async function migrateDb(): Promise { + const config = container.resolve(APP_CONFIG); + const migrationClient = postgres(config.POSTGRES_DB_URI, { max: 1 }); + const migrationDatabase = drizzle(migrationClient, { schema }); + + try { + await migrate(migrationDatabase, { migrationsFolder: config.DRIZZLE_MIGRATIONS_FOLDER }); + } finally { + await migrationClient.end(); + } +} diff --git a/apps/chain-indexer/src/providers/index.ts b/apps/chain-indexer/src/providers/index.ts new file mode 100644 index 0000000000..31a971f1cb --- /dev/null +++ b/apps/chain-indexer/src/providers/index.ts @@ -0,0 +1,5 @@ +export * from "./raw-app-config.provider"; +export * from "./app-config.provider"; +export * from "./logging.provider"; +export * from "./type-registry.provider"; +export * from "./db.provider"; diff --git a/apps/chain-indexer/src/providers/logging.provider.ts b/apps/chain-indexer/src/providers/logging.provider.ts new file mode 100644 index 0000000000..b4b420e1ca --- /dev/null +++ b/apps/chain-indexer/src/providers/logging.provider.ts @@ -0,0 +1,13 @@ +import { LoggerService as LoggerServiceOriginal } from "@akashnetwork/logging"; +import { HttpLoggerInterceptor } from "@akashnetwork/logging/hono"; +import { collectOtel, createOtelLogger } from "@akashnetwork/logging/otel"; +import { container, injectable } from "tsyringe"; + +container.register(HttpLoggerInterceptor, { useValue: new HttpLoggerInterceptor(createOtelLogger({ context: "HTTP" })) }); + +@injectable() +export class LoggerService extends LoggerServiceOriginal { + constructor() { + super({ mixin: collectOtel }); + } +} diff --git a/apps/chain-indexer/src/providers/raw-app-config.provider.ts b/apps/chain-indexer/src/providers/raw-app-config.provider.ts new file mode 100644 index 0000000000..5db0b6d6cc --- /dev/null +++ b/apps/chain-indexer/src/providers/raw-app-config.provider.ts @@ -0,0 +1,8 @@ +import { container, type InjectionToken, instancePerContainerCachingFactory } from "tsyringe"; + +export type RawAppConfig = Record; +export const RAW_APP_CONFIG: InjectionToken = Symbol("RAW_APP_CONFIG"); + +container.register(RAW_APP_CONFIG, { + useFactory: instancePerContainerCachingFactory(() => process.env) +}); diff --git a/apps/chain-indexer/src/providers/type-registry.provider.ts b/apps/chain-indexer/src/providers/type-registry.provider.ts new file mode 100644 index 0000000000..7e43ec3804 --- /dev/null +++ b/apps/chain-indexer/src/providers/type-registry.provider.ts @@ -0,0 +1,36 @@ +import * as v1 from "@akashnetwork/chain-sdk/private-types/akash.v1"; +import * as v1beta4 from "@akashnetwork/chain-sdk/private-types/akash.v1beta4"; +import * as v1beta5 from "@akashnetwork/chain-sdk/private-types/akash.v1beta5"; +import * as cosmosv1 from "@akashnetwork/chain-sdk/private-types/cosmos.v1"; +import * as cosmosv1alpha1 from "@akashnetwork/chain-sdk/private-types/cosmos.v1alpha1"; +import * as cosmosv1beta1 from "@akashnetwork/chain-sdk/private-types/cosmos.v1beta1"; +import * as cosmosv2alpha1 from "@akashnetwork/chain-sdk/private-types/cosmos.v2alpha1"; +import type { GeneratedType } from "@cosmjs/proto-signing"; +import { Registry } from "@cosmjs/proto-signing"; +import { defaultRegistryTypes as stargateDefaultRegistryTypes } from "@cosmjs/stargate"; +import type { InjectionToken } from "tsyringe"; +import { container, inject } from "tsyringe"; + +type DefaultRegistryType = [string, GeneratedType]; +const ibcTypes: DefaultRegistryType[] = stargateDefaultRegistryTypes.filter(([type]) => type.startsWith("/ibc")); +const defaultRegistryTypes: DefaultRegistryType[] = [ + ...Object.values(cosmosv1), + ...Object.values(cosmosv1beta1), + ...Object.values(cosmosv1alpha1), + ...Object.values(cosmosv2alpha1) +] + .filter(x => x && "$type" in x) + .map(x => ["/" + x.$type, x as unknown as GeneratedType]) + .concat(ibcTypes) as DefaultRegistryType[]; + +const akashTypes: ReadonlyArray<[string, GeneratedType]> = [...Object.values(v1), ...Object.values(v1beta4), ...Object.values(v1beta5)] + .filter(x => "$type" in x) + .map(x => ["/" + x.$type, x as unknown as GeneratedType]); + +const registry = new Registry([...defaultRegistryTypes, ...akashTypes]); + +export const TYPE_REGISTRY: InjectionToken = Symbol("TYPE_REGISTRY"); +export type { Registry }; + +container.register(TYPE_REGISTRY, { useValue: registry }); +export const InjectTypeRegistry = (): ReturnType => inject(TYPE_REGISTRY); diff --git a/apps/chain-indexer/src/routes/healthz/healthz.router.ts b/apps/chain-indexer/src/routes/healthz/healthz.router.ts new file mode 100644 index 0000000000..090ede0357 --- /dev/null +++ b/apps/chain-indexer/src/routes/healthz/healthz.router.ts @@ -0,0 +1,26 @@ +import { HealthzResponseSchema } from "@src/http-schemas/healthz.schema"; +import { createRoute } from "@src/lib/create-route/create-route"; +import { OpenApiHonoHandler } from "@src/services/open-api-hono-handler/open-api-hono-handler"; + +export const healthzRouter = new OpenApiHonoHandler(); + +const healthzRoute = createRoute({ + method: "get", + path: "/v1/healthz", + summary: "Health check", + tags: ["Healthz"], + responses: { + 200: { + description: "Returns ok", + content: { + "application/json": { + schema: HealthzResponseSchema + } + } + } + } +}); + +healthzRouter.openapi(healthzRoute, async function routeGetHealthz(c) { + return c.json({ data: { status: "ok" as const } }, 200); +}); diff --git a/apps/chain-indexer/src/routes/index.ts b/apps/chain-indexer/src/routes/index.ts new file mode 100644 index 0000000000..c8bb1f62de --- /dev/null +++ b/apps/chain-indexer/src/routes/index.ts @@ -0,0 +1,2 @@ +export * from "@src/routes/healthz/healthz.router"; +export * from "@src/routes/status/status.router"; diff --git a/apps/chain-indexer/src/routes/status/status.router.ts b/apps/chain-indexer/src/routes/status/status.router.ts new file mode 100644 index 0000000000..0fc48a9da8 --- /dev/null +++ b/apps/chain-indexer/src/routes/status/status.router.ts @@ -0,0 +1,29 @@ +import { container } from "tsyringe"; + +import { StatusResponseSchema } from "@src/http-schemas/status.schema"; +import { createRoute } from "@src/lib/create-route/create-route"; +import { OpenApiHonoHandler } from "@src/services/open-api-hono-handler/open-api-hono-handler"; +import { StatusService } from "@src/services/status/status.service"; + +export const statusRouter = new OpenApiHonoHandler(); + +const statusRoute = createRoute({ + method: "get", + path: "/v1/status", + summary: "Indexer status with per-stream checkpoints", + tags: ["Status"], + responses: { + 200: { + description: "Returns the indexer role, network, and checkpoints", + content: { + "application/json": { + schema: StatusResponseSchema + } + } + } + } +}); + +statusRouter.openapi(statusRoute, async function routeGetStatus(c) { + return c.json(await container.resolve(StatusService).getStatus(), 200); +}); diff --git a/apps/chain-indexer/src/rpc/rpc-client-pool.service.spec.ts b/apps/chain-indexer/src/rpc/rpc-client-pool.service.spec.ts new file mode 100644 index 0000000000..fe43228414 --- /dev/null +++ b/apps/chain-indexer/src/rpc/rpc-client-pool.service.spec.ts @@ -0,0 +1,104 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import { envSchema } from "@src/config/env.config"; +import type { LoggerService } from "@src/providers/logging.provider"; +import { RpcClientPool } from "@src/rpc/rpc-client-pool.service"; + +describe(RpcClientPool.name, () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("returns the result payload from the first healthy node", async () => { + const { pool, fetchMock } = setup(); + fetchMock.mockResolvedValue(jsonResponse({ result: { sync_info: { latest_block_height: "42" } } })); + + const status = await pool.getStatus(); + + expect(status.sync_info.latest_block_height).toBe("42"); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock.mock.calls[0][0]).toBe("http://node-a/status"); + }); + + it("fails over to the next node when the first one errors", async () => { + const { pool, fetchMock } = setup(); + fetchMock.mockRejectedValueOnce(new Error("connection refused")); + fetchMock.mockResolvedValueOnce(jsonResponse({ result: { sync_info: { latest_block_height: "43" } } })); + + const status = await pool.getStatus(); + + expect(status.sync_info.latest_block_height).toBe("43"); + expect(fetchMock.mock.calls.map(call => call[0])).toEqual(["http://node-a/status", "http://node-b/status"]); + }); + + it("keeps a failed node on cooldown for subsequent requests", async () => { + const { pool, fetchMock } = setup(); + fetchMock.mockRejectedValueOnce(new Error("connection refused")); + fetchMock.mockResolvedValue(jsonResponse({ result: { sync_info: { latest_block_height: "44" } } })); + + await pool.getStatus(); + await pool.getStatus(); + + expect(fetchMock.mock.calls.map(call => call[0])).toEqual(["http://node-a/status", "http://node-b/status", "http://node-b/status"]); + }); + + it("treats non-200 responses as node failures", async () => { + const { pool, fetchMock } = setup(); + fetchMock.mockResolvedValueOnce(jsonResponse({}, 502)); + fetchMock.mockResolvedValueOnce(jsonResponse({ result: { sync_info: { latest_block_height: "45" } } })); + + const status = await pool.getStatus(); + + expect(status.sync_info.latest_block_height).toBe("45"); + }); + + it("treats rpc error envelopes as node failures", async () => { + const { pool, fetchMock } = setup(); + fetchMock.mockResolvedValueOnce(jsonResponse({ error: { code: -32603, message: "height not available" } })); + fetchMock.mockResolvedValueOnce(jsonResponse({ result: { sync_info: { latest_block_height: "46" } } })); + + const status = await pool.getStatus(); + + expect(status.sync_info.latest_block_height).toBe("46"); + }); + + it("throws an aggregate error when every node fails", async () => { + const { pool, fetchMock } = setup(); + fetchMock.mockRejectedValue(new Error("connection refused")); + + await expect(pool.getStatus()).rejects.toThrow(AggregateError); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it("requests block and block results with the height as a query parameter", async () => { + const { pool, fetchMock } = setup(); + fetchMock.mockResolvedValue(jsonResponse({ result: { height: "7" } })); + + await pool.getBlock(7); + await pool.getBlockResults(7); + + expect(fetchMock.mock.calls.map(call => call[0])).toEqual(["http://node-a/block?height=7", "http://node-a/block_results?height=7"]); + }); + + function setup() { + const fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + + const config = envSchema.parse({ + POSTGRES_DB_URI: "postgres://unit:unit@localhost:5432/unit", + RPC_NODE_ENDPOINTS: "http://node-a,http://node-b", + RPC_TIMEOUT_MS: 500 + }); + const pool = new RpcClientPool(config, mock()); + return { pool, fetchMock }; + } + + function jsonResponse(payload: unknown, status = 200) { + return { + ok: status >= 200 && status < 300, + status, + json: async () => payload + }; + } +}); diff --git a/apps/chain-indexer/src/rpc/rpc-client-pool.service.ts b/apps/chain-indexer/src/rpc/rpc-client-pool.service.ts new file mode 100644 index 0000000000..60587e5b35 --- /dev/null +++ b/apps/chain-indexer/src/rpc/rpc-client-pool.service.ts @@ -0,0 +1,100 @@ +import { netConfig } from "@akashnetwork/net"; +import { inject, singleton } from "tsyringe"; + +import type { EnvConfig } from "@src/config/env.config"; +import { APP_CONFIG } from "@src/providers/app-config.provider"; +import { LoggerService } from "@src/providers/logging.provider"; +import type { RpcBlockResult, RpcBlockResultsResult, RpcStatusResult } from "@src/rpc/rpc-types"; + +interface RpcNodeState { + endpoint: string; + inFlight: number; + unhealthyUntil: number; +} + +interface RpcEnvelope { + result?: T; + error?: unknown; +} + +@singleton() +export class RpcClientPool { + readonly #nodes: RpcNodeState[]; + readonly #timeoutMs: number; + readonly #cooldownMs: number; + readonly #logger: LoggerService; + + constructor(@inject(APP_CONFIG) config: EnvConfig, @inject(LoggerService) logger: LoggerService) { + const endpoints = config.RPC_NODE_ENDPOINTS + ? config.RPC_NODE_ENDPOINTS.split(",") + .map(endpoint => endpoint.trim()) + .filter(Boolean) + : netConfig.getAllBaseRpcUrls(config.NETWORK); + + if (endpoints.length === 0) { + throw new Error(`No RPC endpoints available for network ${config.NETWORK}`); + } + + this.#nodes = endpoints.map(endpoint => ({ endpoint: endpoint.replace(/\/$/, ""), inFlight: 0, unhealthyUntil: 0 })); + this.#timeoutMs = config.RPC_TIMEOUT_MS; + this.#cooldownMs = config.RPC_NODE_COOLDOWN_MS; + this.#logger = logger; + this.#logger.setContext("RPC_POOL"); + } + + async getStatus(): Promise { + return await this.#get("/status"); + } + + async getBlock(height: number): Promise { + return await this.#get(`/block?height=${height}`); + } + + async getBlockResults(height: number): Promise { + return await this.#get(`/block_results?height=${height}`); + } + + async #get(path: string): Promise { + const errors: unknown[] = []; + + for (const node of this.#candidates()) { + node.inFlight++; + try { + const result = await this.#fetchFromNode(node.endpoint, path); + node.unhealthyUntil = 0; + return result; + } catch (error) { + errors.push(error); + node.unhealthyUntil = Date.now() + this.#cooldownMs; + this.#logger.warn({ event: "RPC_NODE_FAILED", endpoint: node.endpoint, path, error }); + } finally { + node.inFlight--; + } + } + + throw new AggregateError(errors, `All RPC nodes failed for ${path}`); + } + + async #fetchFromNode(endpoint: string, path: string): Promise { + const response = await fetch(`${endpoint}${path}`, { signal: AbortSignal.timeout(this.#timeoutMs) }); + + if (!response.ok) { + throw new Error(`RPC responded with status ${response.status}`); + } + + const envelope = (await response.json()) as RpcEnvelope; + + if (envelope.error || envelope.result === undefined) { + throw new Error(`RPC returned an error payload: ${JSON.stringify(envelope.error ?? "empty result")}`); + } + + return envelope.result; + } + + #candidates(): RpcNodeState[] { + const now = Date.now(); + const healthy = this.#nodes.filter(node => node.unhealthyUntil <= now); + const pool = healthy.length > 0 ? healthy : this.#nodes; + return [...pool].sort((a, b) => a.inFlight - b.inFlight); + } +} diff --git a/apps/chain-indexer/src/rpc/rpc-types.ts b/apps/chain-indexer/src/rpc/rpc-types.ts new file mode 100644 index 0000000000..54d3f4969c --- /dev/null +++ b/apps/chain-indexer/src/rpc/rpc-types.ts @@ -0,0 +1,36 @@ +export interface RpcStatusResult { + sync_info: { + latest_block_height: string; + }; +} + +export interface RpcBlockResult { + block_id: { + hash: string; + }; + block: { + header: { + height: string; + time: string; + proposer_address: string; + last_block_id?: { + hash: string; + }; + }; + data: { + txs: string[]; + }; + }; +} + +export interface RpcTxResult { + code: number; + log?: string; + gas_used: string; + gas_wanted: string; +} + +export interface RpcBlockResultsResult { + height: string; + txs_results: RpcTxResult[] | null; +} diff --git a/apps/chain-indexer/src/server.ts b/apps/chain-indexer/src/server.ts new file mode 100644 index 0000000000..f64e65c9d3 --- /dev/null +++ b/apps/chain-indexer/src/server.ts @@ -0,0 +1,6 @@ +import { bootstrap } from "./index"; + +void bootstrap().catch(error => { + console.error("Failed to bootstrap chain-indexer", error); + process.exitCode = 1; +}); diff --git a/apps/chain-indexer/src/services/app-config/app-config.service.ts b/apps/chain-indexer/src/services/app-config/app-config.service.ts new file mode 100644 index 0000000000..ffa5eaeab3 --- /dev/null +++ b/apps/chain-indexer/src/services/app-config/app-config.service.ts @@ -0,0 +1,12 @@ +import { inject, singleton } from "tsyringe"; + +import type { EnvConfig, envSchema } from "@src/config/env.config"; +import { APP_CONFIG } from "@src/providers/app-config.provider"; +import { ConfigService } from "@src/services/config/config.service"; + +@singleton() +export class AppConfigService extends ConfigService { + constructor(@inject(APP_CONFIG) config: EnvConfig) { + super({ config }); + } +} diff --git a/apps/chain-indexer/src/services/config/config.service.ts b/apps/chain-indexer/src/services/config/config.service.ts new file mode 100644 index 0000000000..4057e6bccc --- /dev/null +++ b/apps/chain-indexer/src/services/config/config.service.ts @@ -0,0 +1,19 @@ +import type { z, ZodEffects, ZodObject, ZodRawShape } from "zod"; + +interface ConfigServiceOptions | ZodEffects>, C extends Record> { + config?: [C] extends [Record] ? z.infer : z.infer & C; +} + +export class ConfigService | ZodEffects>, C extends Record = Record> { + readonly #config: C & z.infer; + + constructor(options: ConfigServiceOptions) { + this.#config = { + ...options.config + } as C & z.infer; + } + + get)>(key: K): (C & z.infer)[K] { + return this.#config[key]; + } +} diff --git a/apps/chain-indexer/src/services/hono-error-handler/hono-error-handler.service.ts b/apps/chain-indexer/src/services/hono-error-handler/hono-error-handler.service.ts new file mode 100644 index 0000000000..4654e46aee --- /dev/null +++ b/apps/chain-indexer/src/services/hono-error-handler/hono-error-handler.service.ts @@ -0,0 +1,102 @@ +import { createOtelLogger } from "@akashnetwork/logging/otel"; +import { HTTPException } from "hono/http-exception"; +import { isHttpError } from "http-errors"; +import { singleton } from "tsyringe"; +import { ZodError } from "zod"; + +import type { AppContext } from "@src/types/app-context"; + +@singleton() +export class HonoErrorHandlerService { + readonly #logger = createOtelLogger({ context: "ErrorHandler" }); + + constructor() { + this.handle = this.handle.bind(this); + } + + async handle(error: unknown, c: AppContext): Promise { + this.#logger.error(error); + + if (error instanceof HTTPException) { + return c.json( + { + error: error.name || "HTTPException", + message: error.message, + code: this.#getErrorCode(error), + type: this.#getErrorType(error) + }, + { status: error.status } + ); + } + + if (isHttpError(error)) { + const { name } = error.constructor; + return c.json( + { + error: name, + message: error.message, + code: this.#getErrorCode(error), + type: this.#getErrorType(error), + data: error.data + }, + { status: error.status } + ); + } + + if (error instanceof ZodError) { + return c.json( + { + error: "BadRequestError", + message: "Validation error", + code: "validation_error", + type: "validation_error", + data: error.errors + }, + { status: 400 } + ); + } + + const message = error instanceof Error ? error.message : "Internal server error"; + return c.json( + { + error: "InternalServerError", + message, + code: "internal_server_error", + type: "server_error" + }, + { status: 500 } + ); + } + + #getErrorCode(error: { status?: number }): string { + switch (error.status) { + case 400: + return "bad_request"; + case 401: + return "unauthorized"; + case 403: + return "forbidden"; + case 404: + return "not_found"; + case 409: + return "conflict"; + case 429: + return "rate_limited"; + case 502: + case 503: + return "service_unavailable"; + default: + return "unknown_error"; + } + } + + #getErrorType(error: { status?: number }): string { + if (error.status && error.status >= 500) { + return "server_error"; + } + if (error.status && error.status >= 400) { + return "client_error"; + } + return "unknown_error"; + } +} diff --git a/apps/chain-indexer/src/services/open-api-hono-handler/open-api-hono-handler.ts b/apps/chain-indexer/src/services/open-api-hono-handler/open-api-hono-handler.ts new file mode 100644 index 0000000000..3f6a6ed249 --- /dev/null +++ b/apps/chain-indexer/src/services/open-api-hono-handler/open-api-hono-handler.ts @@ -0,0 +1,26 @@ +import type { OpenAPIHonoOptions } from "@hono/zod-openapi"; +import { OpenAPIHono } from "@hono/zod-openapi"; +import type { Env, Hono, Schema } from "hono"; +import { container } from "tsyringe"; + +import { HonoErrorHandlerService } from "@src/services/hono-error-handler/hono-error-handler.service"; +import type { AppContext, AppEnv } from "@src/types/app-context"; + +type HonoInit = ConstructorParameters[0] & OpenAPIHonoOptions; + +export class OpenApiHonoHandler, BasePath extends string = "/"> extends OpenAPIHono< + E, + S, + BasePath +> { + constructor(init?: Omit, "defaultHook">) { + super({ + ...init, + defaultHook: (result, c) => { + if (!result.success && "error" in result) { + return container.resolve(HonoErrorHandlerService).handle(result.error, c as unknown as AppContext); + } + } + }); + } +} diff --git a/apps/chain-indexer/src/services/shutdown-server/shutdown-server.ts b/apps/chain-indexer/src/services/shutdown-server/shutdown-server.ts new file mode 100644 index 0000000000..0b62a13cf7 --- /dev/null +++ b/apps/chain-indexer/src/services/shutdown-server/shutdown-server.ts @@ -0,0 +1,31 @@ +import type { Logger } from "@akashnetwork/logging"; +import type { ServerType } from "@hono/node-server"; + +export async function shutdownServer(server: ServerType, appLogger: Logger, onShutdown?: () => void | Promise): Promise { + return new Promise(resolve => { + const shutdown = (error?: unknown) => { + if (error) { + appLogger.error({ event: "SERVER_CLOSE_ERROR", error }); + } + + Promise.resolve() + .then(() => onShutdown?.()) + .catch(onShutdownError => { + appLogger.error({ event: "ON_SHUTDOWN_ERROR", error: onShutdownError }); + }) + .finally(() => { + resolve(); + }); + }; + + try { + if (server.listening) { + server.close(shutdown); + } else { + shutdown(); + } + } catch (error) { + shutdown(error); + } + }); +} diff --git a/apps/chain-indexer/src/services/start-server/start-server.ts b/apps/chain-indexer/src/services/start-server/start-server.ts new file mode 100644 index 0000000000..0fc9e6b29e --- /dev/null +++ b/apps/chain-indexer/src/services/start-server/start-server.ts @@ -0,0 +1,57 @@ +import type { LoggerService } from "@akashnetwork/logging"; +import type { ServerType } from "@hono/node-server"; +import { serve } from "@hono/node-server"; +import type EventEmitter from "events"; +import type { Env, Hono } from "hono"; +import once from "lodash/once"; +import type { DependencyContainer } from "tsyringe"; +import { container as rootContainer } from "tsyringe"; + +import { shutdownServer } from "@src/services/shutdown-server/shutdown-server"; + +export async function startServer( + app: Hono, + logger: LoggerService, + processEvents: EventEmitter, + options: { + port: number; + beforeStart?: () => Promise; + container?: DependencyContainer; + } +): Promise { + const container = options.container ?? rootContainer; + const disposeContainerOnce = once(() => { + logger.info({ event: "DISPOSING_CONTAINER" }); + return Promise.resolve(container.dispose()).catch(error => { + logger.error({ event: "CONTAINER_DISPOSE_ERROR", error }); + }); + }); + + let server: ServerType | undefined; + const shutdown = once(async (reason: string) => { + logger.info({ event: "APP_SERVER_SHUTDOWN_REQUESTED", reason }); + if (server) { + await shutdownServer(server, logger, disposeContainerOnce); + } else { + await disposeContainerOnce(); + } + }); + try { + await options.beforeStart?.(); + + logger.info({ event: "SERVER_STARTING", url: `http://localhost:${options.port}`, NODE_OPTIONS: process.env.NODE_OPTIONS }); + server = serve({ + fetch: app.fetch, + port: options.port + }); + + server.on("close", disposeContainerOnce); + processEvents.on("SIGTERM", () => shutdown("SIGTERM")); + processEvents.on("SIGINT", () => shutdown("SIGINT")); + processEvents.on("exit", exitCode => shutdown(`EXIT:${exitCode}`)); + return server; + } catch (error) { + logger.error({ event: "SERVER_START_ERROR", error }); + await shutdown("SERVER_START_ERROR"); + } +} diff --git a/apps/chain-indexer/src/services/status/status.service.ts b/apps/chain-indexer/src/services/status/status.service.ts new file mode 100644 index 0000000000..82a11812de --- /dev/null +++ b/apps/chain-indexer/src/services/status/status.service.ts @@ -0,0 +1,35 @@ +import { inject, singleton } from "tsyringe"; + +import type { EnvConfig } from "@src/config/env.config"; +import { IndexerState } from "@src/db/schema"; +import type { StatusResponse } from "@src/http-schemas/status.schema"; +import { APP_CONFIG } from "@src/providers/app-config.provider"; +import type { ChainDatabase } from "@src/providers/db.provider"; +import { CHAIN_DB } from "@src/providers/db.provider"; + +@singleton() +export class StatusService { + readonly #db: ChainDatabase; + readonly #config: EnvConfig; + + constructor(@inject(CHAIN_DB) db: ChainDatabase, @inject(APP_CONFIG) config: EnvConfig) { + this.#db = db; + this.#config = config; + } + + async getStatus(): Promise { + const checkpoints = await this.#db.select().from(IndexerState); + + return { + data: { + network: this.#config.NETWORK, + role: this.#config.INDEXER_ROLE, + checkpoints: checkpoints.map(checkpoint => ({ + stream: checkpoint.stream, + lastHeight: checkpoint.lastHeight, + updatedAt: checkpoint.updatedAt.toISOString() + })) + } + }; + } +} diff --git a/apps/chain-indexer/src/types/app-context.ts b/apps/chain-indexer/src/types/app-context.ts new file mode 100644 index 0000000000..55904ac4eb --- /dev/null +++ b/apps/chain-indexer/src/types/app-context.ts @@ -0,0 +1,7 @@ +import type { Context, Env, Input } from "hono"; + +export interface AppContext> extends Context {} + +export interface AppEnv extends Env { + Variables: Env["Variables"]; +} diff --git a/apps/chain-indexer/test/setup-unit-env.ts b/apps/chain-indexer/test/setup-unit-env.ts new file mode 100644 index 0000000000..fded23ac1e --- /dev/null +++ b/apps/chain-indexer/test/setup-unit-env.ts @@ -0,0 +1 @@ +import "reflect-metadata"; diff --git a/apps/chain-indexer/test/setup-unit-tests.ts b/apps/chain-indexer/test/setup-unit-tests.ts new file mode 100644 index 0000000000..058eb16608 --- /dev/null +++ b/apps/chain-indexer/test/setup-unit-tests.ts @@ -0,0 +1,10 @@ +import { container } from "tsyringe"; +import { afterAll } from "vitest"; + +afterAll(async () => { + try { + await container.dispose(); + } catch { + /** container may already be disposed in tests */ + } +}); diff --git a/apps/chain-indexer/tsconfig.build.json b/apps/chain-indexer/tsconfig.build.json new file mode 100644 index 0000000000..abc725c59d --- /dev/null +++ b/apps/chain-indexer/tsconfig.build.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "baseUrl": ".", + "noImplicitAny": true, + "paths": { + "@src/*": ["./src/*"], + "@test/*": ["./test/*"] + }, + "strict": true + }, + "extends": "@akashnetwork/dev-config/tsconfig.base-node.json" +} diff --git a/apps/chain-indexer/tsconfig.json b/apps/chain-indexer/tsconfig.json new file mode 100644 index 0000000000..a9d7bde952 --- /dev/null +++ b/apps/chain-indexer/tsconfig.json @@ -0,0 +1,13 @@ +{ + "exclude": ["node_modules", "dist", "test", "src/**/*.spec.ts"], + "extends": "./tsconfig.build.json", + "include": ["src/**/*"], + "compilerOptions": { + "target": "es2022", + "useDefineForClassFields": false, + "module": "NodeNext", + "lib": ["esnext"], + "allowImportingTsExtensions": true, + "rewriteRelativeImportExtensions": true + } +} diff --git a/apps/chain-indexer/tsup.config.ts b/apps/chain-indexer/tsup.config.ts new file mode 100644 index 0000000000..fb175af820 --- /dev/null +++ b/apps/chain-indexer/tsup.config.ts @@ -0,0 +1,24 @@ +import { applyDefaults } from "@akashnetwork/dev-config/tsup-plugins.ts"; +import { fileURLToPath } from "node:url"; +import { defineConfig } from "tsup"; + +import packageJson from "./package.json"; +import tsconfig from "./tsconfig.json"; + +const isProduction = process.env.NODE_ENV === "production"; + +export default defineConfig(async overrideOptions => + applyDefaults({ + packageJson, + prependEffectsToEntries: ["reflect-metadata", "@akashnetwork/env-loader"], + entry: { + server: "./src/server.ts", + instrumentation: fileURLToPath(import.meta.resolve("@akashnetwork/instrumentation/register")) + }, + target: tsconfig.compilerOptions.target, + tsconfig: "tsconfig.build.json", + external: ["pino-pretty"], + onSuccess: overrideOptions.watch && !isProduction ? "npm run prod" : undefined, + ...overrideOptions + }) +); diff --git a/apps/chain-indexer/vitest.config.ts b/apps/chain-indexer/vitest.config.ts new file mode 100644 index 0000000000..202b2cea50 --- /dev/null +++ b/apps/chain-indexer/vitest.config.ts @@ -0,0 +1,34 @@ +import path from "path"; +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + resolve: { + alias: { + "@src": path.resolve(__dirname, "./src"), + "@test": path.resolve(__dirname, "./test") + } + }, + test: { + environment: "node", + globals: false, + testTimeout: 10_000, + coverage: { + provider: "v8", + include: ["src/**/*.ts"], + exclude: ["src/**/*.spec.ts", "src/**/*.d.ts", "src/**/index.ts"], + reportsDirectory: "./coverage" + }, + reporters: ["default", "junit"], + outputFile: { junit: "junit.xml" }, + projects: [ + { + extends: true, + test: { + name: "unit", + include: ["src/**/*.spec.ts"], + setupFiles: ["./test/setup-unit-env.ts", "./test/setup-unit-tests.ts"] + } + } + ] + } +}); diff --git a/package-lock.json b/package-lock.json index efa52f5a13..7443a4815f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -531,6 +531,80 @@ } } }, + "apps/chain-indexer": { + "name": "@akashnetwork/chain-indexer", + "version": "0.0.1", + "license": "Apache-2.0", + "dependencies": { + "@akashnetwork/chain-sdk": "1.0.0-alpha.41", + "@akashnetwork/env-loader": "*", + "@akashnetwork/instrumentation": "*", + "@akashnetwork/logging": "*", + "@akashnetwork/net": "*", + "@cosmjs/encoding": "~0.38.0", + "@cosmjs/proto-signing": "~0.38.0", + "@cosmjs/stargate": "~0.38.0", + "@hono/node-server": "1.13.7", + "@hono/otel": "~0.4.0", + "@hono/zod-openapi": "0.18.4", + "@opentelemetry/api": "^1.9.0", + "drizzle-orm": "^0.45.2", + "hono": "4.6.12", + "http-errors": "^2.0.0", + "lodash": "^4.17.21", + "postgres": "^3.4.4", + "reflect-metadata": "^0.2.2", + "tsyringe": "^4.10.0", + "zod": "3.*" + }, + "devDependencies": { + "@akashnetwork/dev-config": "*", + "@types/lodash": "^4.17.0", + "@types/node": "^22.13.11", + "@typescript-eslint/eslint-plugin": "^8.64.0", + "@vitest/coverage-v8": "^4.1.5", + "cosmjs-types": "~0.11.0", + "drizzle-kit": "^0.31.10", + "eslint": "^9.39.5", + "eslint-config-next": "^15.5.20", + "eslint-plugin-simple-import-sort": "^13.0.0", + "prettier": "^3.3.0", + "tsup": "^8.5.1", + "typescript": "~5.8.2", + "vitest": "^4.1.5", + "vitest-mock-extended": "^4.0.0" + } + }, + "apps/chain-indexer/node_modules/@cosmjs/encoding": { + "version": "0.38.1", + "resolved": "https://registry.npmjs.org/@cosmjs/encoding/-/encoding-0.38.1.tgz", + "integrity": "sha512-i5jGgJhiXs7boePGA48xzYxnCbf3MS5nT/R2HvnvSQyVAG2A99NyL96lBgmz6etOJSGOiwVrB8uh1Qp5bq6WKQ==", + "license": "Apache-2.0", + "dependencies": { + "@scure/base": "^2.0.0", + "base64-js": "^1.3.0", + "readonly-date-esm": "^2.0.0" + } + }, + "apps/chain-indexer/node_modules/@scure/base": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@scure/base/-/base-2.2.0.tgz", + "integrity": "sha512-b8XEupJibegiXV+tDUseI8oLQc8ei3d/4Jkb2RpbHh3MfE054ov3uIz2dhFkB3FI8iwYkEh0gGCApkrYggkPNg==", + "license": "MIT", + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "apps/chain-indexer/node_modules/cosmjs-types": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/cosmjs-types/-/cosmjs-types-0.11.0.tgz", + "integrity": "sha512-kDSkgHpRTrg1413jCNehT3P21+EBxZWFMBr9JEzVfmPiNdtuwAoLAkCYo7c7i/pTakAwyHsXbxOg8kkD+AN33w==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=20.19" + } + }, "apps/deploy-web": { "name": "@akashnetwork/console-web", "version": "3.31.0", @@ -5213,6 +5287,10 @@ "@grpc/grpc-js": "^1.10.6" } }, + "node_modules/@akashnetwork/chain-indexer": { + "resolved": "apps/chain-indexer", + "link": true + }, "node_modules/@akashnetwork/chain-sdk": { "version": "1.0.0-alpha.41", "resolved": "https://registry.npmjs.org/@akashnetwork/chain-sdk/-/chain-sdk-1.0.0-alpha.41.tgz", @@ -21477,6 +21555,7 @@ "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.2.0.tgz", "integrity": "sha512-8yQrvS6sMpSwIovhPOwfyNf2Wz6v/B62LFSVYQ85+Rq3tLsBIG7rP5geMxaijTUxSkrO6RzN/IRuIAADYQsleA==", "license": "MIT", + "peer": true, "dependencies": { "@types/react": "*" } From 7141fc7afc6d1b3fb094e128bc5010a7787e6b13 Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Tue, 11 Aug 2026 18:17:03 +0530 Subject: [PATCH 02/14] fix(indexer): address chain-indexer scaffold review findings - fail bootstrap when the HTTP server cannot start and exit the process when the sync loop dies so healthz cannot report healthy on a dead sync - detect advisory-lock session loss via backend pid checks to prevent two sync leaders after a transparent reconnect - retry transient sync errors with capped backoff; continuity breaks and leadership loss stay fatal - serialize migrations with a pg advisory lock - select existing message types before inserting to stop id sequence burn; widen message_types.id to serial and messages.type_id to integer - fail fast when txs_results does not match block txs instead of persisting txs as successful with zero gas - serialize Buffers as base64 in canonical json despite Buffer#toJSON - treat empty SYNC_START_HEIGHT as absent; validate PORT range - force-close keep-alive sockets on shutdown; handle async listen errors; drop the no-op exit listener - return a constant message for unexpected errors; log via LoggerService - remove dead InjectTypeRegistry export and inline comments --- ...owlett.sql => 0000_remarkable_scourge.sql} | 4 +- .../drizzle/meta/0000_snapshot.json | 6 +- apps/chain-indexer/drizzle/meta/_journal.json | 4 +- .../src/config/env.config.spec.ts | 37 ++++++++ apps/chain-indexer/src/config/env.config.ts | 7 +- apps/chain-indexer/src/db/schema.ts | 6 +- apps/chain-indexer/src/index.ts | 13 ++- .../pipeline/block-committer.service.spec.ts | 89 +++++++++++++++++++ .../src/pipeline/block-committer.service.ts | 38 ++++++-- .../pipeline/block-decoder.service.spec.ts | 9 ++ .../src/pipeline/block-decoder.service.ts | 12 ++- .../src/pipeline/canonical-json.spec.ts | 4 + .../src/pipeline/canonical-json.ts | 13 +-- .../src/pipeline/sync-runner.service.ts | 72 +++++++++++++-- .../src/providers/db.provider.ts | 4 + .../src/providers/type-registry.provider.ts | 3 +- apps/chain-indexer/src/rpc/rpc-types.ts | 7 +- apps/chain-indexer/src/server.ts | 4 +- .../hono-error-handler.service.ts | 3 +- .../shutdown-server/shutdown-server.ts | 14 +++ .../src/services/start-server/start-server.ts | 35 ++++++-- apps/chain-indexer/test/setup-unit-tests.ts | 6 +- 22 files changed, 334 insertions(+), 56 deletions(-) rename apps/chain-indexer/drizzle/{0000_nappy_james_howlett.sql => 0000_remarkable_scourge.sql} (95%) create mode 100644 apps/chain-indexer/src/config/env.config.spec.ts create mode 100644 apps/chain-indexer/src/pipeline/block-committer.service.spec.ts diff --git a/apps/chain-indexer/drizzle/0000_nappy_james_howlett.sql b/apps/chain-indexer/drizzle/0000_remarkable_scourge.sql similarity index 95% rename from apps/chain-indexer/drizzle/0000_nappy_james_howlett.sql rename to apps/chain-indexer/drizzle/0000_remarkable_scourge.sql index 45923803e8..2232527ccd 100644 --- a/apps/chain-indexer/drizzle/0000_nappy_james_howlett.sql +++ b/apps/chain-indexer/drizzle/0000_remarkable_scourge.sql @@ -16,7 +16,7 @@ CREATE TABLE "indexer_state" ( ); --> statement-breakpoint CREATE TABLE "cosmos"."message_types" ( - "id" "smallserial" PRIMARY KEY NOT NULL, + "id" serial PRIMARY KEY NOT NULL, "type" text NOT NULL ); --> statement-breakpoint @@ -24,7 +24,7 @@ CREATE TABLE "cosmos"."messages" ( "height" bigint NOT NULL, "tx_index" integer NOT NULL, "index" integer NOT NULL, - "type_id" smallint NOT NULL, + "type_id" integer NOT NULL, "body" jsonb, CONSTRAINT "messages_height_tx_index_index_pk" PRIMARY KEY("height","tx_index","index") ); diff --git a/apps/chain-indexer/drizzle/meta/0000_snapshot.json b/apps/chain-indexer/drizzle/meta/0000_snapshot.json index cd0c1d0953..fa55371f96 100644 --- a/apps/chain-indexer/drizzle/meta/0000_snapshot.json +++ b/apps/chain-indexer/drizzle/meta/0000_snapshot.json @@ -1,5 +1,5 @@ { - "id": "6cc7b5d5-bf47-4e13-a098-4819d0a2863e", + "id": "332f29a4-e784-4238-9a9e-7139a844e7e2", "prevId": "00000000-0000-0000-0000-000000000000", "version": "7", "dialect": "postgresql", @@ -90,7 +90,7 @@ "columns": { "id": { "name": "id", - "type": "smallserial", + "type": "serial", "primaryKey": true, "notNull": true }, @@ -149,7 +149,7 @@ }, "type_id": { "name": "type_id", - "type": "smallint", + "type": "integer", "primaryKey": false, "notNull": true }, diff --git a/apps/chain-indexer/drizzle/meta/_journal.json b/apps/chain-indexer/drizzle/meta/_journal.json index 60910f3091..e539d4c4af 100644 --- a/apps/chain-indexer/drizzle/meta/_journal.json +++ b/apps/chain-indexer/drizzle/meta/_journal.json @@ -5,8 +5,8 @@ { "idx": 0, "version": "7", - "when": 1786433105904, - "tag": "0000_nappy_james_howlett", + "when": 1786452233786, + "tag": "0000_remarkable_scourge", "breakpoints": true } ] diff --git a/apps/chain-indexer/src/config/env.config.spec.ts b/apps/chain-indexer/src/config/env.config.spec.ts new file mode 100644 index 0000000000..5bcf5efab2 --- /dev/null +++ b/apps/chain-indexer/src/config/env.config.spec.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; + +import { envSchema } from "@src/config/env.config"; + +describe("envSchema", () => { + it("parses a minimal environment with defaults", () => { + const config = setup(); + + expect(config.INDEXER_ROLE).toBe("sync"); + expect(config.PORT).toBe(3092); + expect(config.SYNC_START_HEIGHT).toBeUndefined(); + }); + + it("treats an empty SYNC_START_HEIGHT as absent", () => { + const config = setup({ SYNC_START_HEIGHT: "" }); + + expect(config.SYNC_START_HEIGHT).toBeUndefined(); + }); + + it("coerces a numeric SYNC_START_HEIGHT string", () => { + const config = setup({ SYNC_START_HEIGHT: "12345" }); + + expect(config.SYNC_START_HEIGHT).toBe(12345); + }); + + it("rejects a PORT above 65535", () => { + expect(() => setup({ PORT: "65536" })).toThrow(); + }); + + it("rejects a fractional PORT", () => { + expect(() => setup({ PORT: "3092.5" })).toThrow(); + }); + + function setup(overrides?: Record) { + return envSchema.parse({ POSTGRES_DB_URI: "postgres://unit:unit@localhost:5432/unit", ...overrides }); + } +}); diff --git a/apps/chain-indexer/src/config/env.config.ts b/apps/chain-indexer/src/config/env.config.ts index 25982e1e9c..eb2ec6e76d 100644 --- a/apps/chain-indexer/src/config/env.config.ts +++ b/apps/chain-indexer/src/config/env.config.ts @@ -1,5 +1,8 @@ import { z } from "zod"; +/** Treats an empty string as absent so `VAR=` lines in env files don't fail coerced-number validation. */ +const emptyStringAsUndefined = (value: unknown) => (value === "" ? undefined : value); + export const envSchema = z.object({ INDEXER_ROLE: z.enum(["sync", "backfill", "api", "jobs"]).default("sync"), NETWORK: z.enum(["mainnet", "sandbox", "testnet"]).default("sandbox"), @@ -9,7 +12,7 @@ export const envSchema = z.object({ RPC_TIMEOUT_MS: z.number({ coerce: true }).int().positive().default(15_000), RPC_NODE_COOLDOWN_MS: z.number({ coerce: true }).int().positive().default(30_000), /** First height to sync when the database has no checkpoint yet. Defaults to the current chain tip. */ - SYNC_START_HEIGHT: z.number({ coerce: true }).int().positive().optional(), + SYNC_START_HEIGHT: z.preprocess(emptyStringAsUndefined, z.number({ coerce: true }).int().positive().optional()), SYNC_POLL_INTERVAL_MS: z.number({ coerce: true }).int().positive().default(3_000), /** Decoded message bodies above this serialized size are stored as null to keep pathological messages out of Postgres. */ MESSAGE_BODY_MAX_BYTES: z.number({ coerce: true }).int().positive().default(65_536), @@ -17,7 +20,7 @@ export const envSchema = z.object({ LOG_LEVEL: z.enum(["fatal", "error", "warn", "info", "debug", "trace"]).optional().default("info"), STD_OUT_LOG_FORMAT: z.enum(["json", "pretty"]).optional().default("json"), NODE_ENV: z.enum(["development", "production", "test"]).optional().default("development"), - PORT: z.number({ coerce: true }).optional().default(3092) + PORT: z.number({ coerce: true }).int().min(1).max(65_535).optional().default(3092) }); export type EnvConfig = z.infer; diff --git a/apps/chain-indexer/src/db/schema.ts b/apps/chain-indexer/src/db/schema.ts index 98c59a8dc6..ea0591d979 100644 --- a/apps/chain-indexer/src/db/schema.ts +++ b/apps/chain-indexer/src/db/schema.ts @@ -1,4 +1,4 @@ -import { bigint, index, integer, jsonb, pgSchema, pgTable, primaryKey, smallint, smallserial, text, timestamp, uniqueIndex } from "drizzle-orm/pg-core"; +import { bigint, index, integer, jsonb, pgSchema, pgTable, primaryKey, serial, text, timestamp, uniqueIndex } from "drizzle-orm/pg-core"; import { bytea } from "@src/db/bytea"; @@ -35,7 +35,7 @@ export const Transactions = cosmosSchema.table( export const MessageTypes = cosmosSchema.table( "message_types", { - id: smallserial("id").primaryKey(), + id: serial("id").primaryKey(), type: text("type").notNull() }, t => [uniqueIndex("message_types_type_idx").on(t.type)] @@ -47,7 +47,7 @@ export const Messages = cosmosSchema.table( height: bigint("height", { mode: "number" }).notNull(), txIndex: integer("tx_index").notNull(), index: integer("index").notNull(), - typeId: smallint("type_id") + typeId: integer("type_id") .notNull() .references(() => MessageTypes.id), body: jsonb("body") diff --git a/apps/chain-indexer/src/index.ts b/apps/chain-indexer/src/index.ts index 14a6eab0a8..3944fc6b74 100644 --- a/apps/chain-indexer/src/index.ts +++ b/apps/chain-indexer/src/index.ts @@ -8,6 +8,7 @@ import { createApp } from "@src/app"; import { SyncRunnerService } from "@src/pipeline/sync-runner.service"; import { migrateDb } from "@src/providers/db.provider"; import { AppConfigService } from "@src/services/app-config/app-config.service"; +import { shutdownServer } from "@src/services/shutdown-server/shutdown-server"; import { startServer } from "@src/services/start-server/start-server"; export async function bootstrap(): Promise { @@ -18,8 +19,16 @@ export async function bootstrap(): Promise { switch (role) { case "sync": { await migrateDb(); - await startServer(createApp(), logger, process, { port: config.get("PORT") }); - await container.resolve(SyncRunnerService).start(); + const server = await startServer(createApp(), logger, process, { port: config.get("PORT") }); + + try { + await container.resolve(SyncRunnerService).start(); + } catch (error) { + logger.error({ event: "SYNC_FATAL", error }); + process.exitCode = 1; + } + + await shutdownServer(server, logger); return; } case "api": { diff --git a/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts b/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts new file mode 100644 index 0000000000..543ca4ab4d --- /dev/null +++ b/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts @@ -0,0 +1,89 @@ +import { describe, expect, it } from "vitest"; + +import { Messages, MessageTypes } from "@src/db/schema"; +import { BlockCommitterService } from "@src/pipeline/block-committer.service"; +import type { DecodedBlock } from "@src/pipeline/decoded-block"; +import type { ChainDatabase } from "@src/providers/db.provider"; + +const MSG_SEND = "/cosmos.bank.v1beta1.MsgSend"; + +describe(BlockCommitterService.name, () => { + it("reuses ids of message types that already exist instead of inserting them", async () => { + const { committer, insertedRows } = setup({ selectResults: [[{ id: 7, type: MSG_SEND }]] }); + + await committer.commit(buildBlock([MSG_SEND])); + + expect(insertedRows.filter(call => call.table === MessageTypes)).toEqual([]); + expect(insertedRows.find(call => call.table === Messages)?.rows).toEqual([expect.objectContaining({ typeId: 7 })]); + }); + + it("inserts only the message types missing from the database and caches returned ids", async () => { + const { committer, insertedRows } = setup({ + selectResults: [[{ id: 7, type: MSG_SEND }]], + insertReturning: [{ id: 8, type: "/akash.deployment.v1beta4.MsgCreateDeployment" }] + }); + + await committer.commit(buildBlock([MSG_SEND, "/akash.deployment.v1beta4.MsgCreateDeployment"])); + + const messageTypeInserts = insertedRows.filter(call => call.table === MessageTypes); + expect(messageTypeInserts).toEqual([{ table: MessageTypes, rows: [{ type: "/akash.deployment.v1beta4.MsgCreateDeployment" }] }]); + expect(insertedRows.find(call => call.table === Messages)?.rows).toEqual([expect.objectContaining({ typeId: 7 }), expect.objectContaining({ typeId: 8 })]); + }); + + it("resolves ids inserted concurrently by another process via a follow-up select", async () => { + const { committer, insertedRows } = setup({ + selectResults: [[], [{ id: 9, type: MSG_SEND }]], + insertReturning: [] + }); + + await committer.commit(buildBlock([MSG_SEND])); + + expect(insertedRows.find(call => call.table === Messages)?.rows).toEqual([expect.objectContaining({ typeId: 9 })]); + }); + + function setup(input?: { selectResults?: Array>; insertReturning?: Array<{ id: number; type: string }> }) { + const selectResults = [...(input?.selectResults ?? [[]])]; + const insertedRows: Array<{ table: unknown; rows: unknown }> = []; + + const dbFake = { + select: () => ({ from: () => ({ where: () => Promise.resolve(selectResults.shift() ?? []) }) }), + insert: (table: unknown) => ({ + values: (rows: unknown) => { + insertedRows.push({ table, rows }); + return { + onConflictDoNothing: () => + Object.assign(Promise.resolve(), { + returning: () => Promise.resolve(input?.insertReturning ?? []) + }), + onConflictDoUpdate: () => Promise.resolve() + }; + } + }), + transaction: (callback: (tx: unknown) => Promise) => callback(dbFake) + }; + + const committer = new BlockCommitterService(dbFake as unknown as ChainDatabase); + return { committer, insertedRows }; + } + + function buildBlock(typeUrls: string[]): DecodedBlock { + return { + height: 10, + datetime: new Date("2026-08-11T00:00:00Z"), + hash: Buffer.from("aa".repeat(32), "hex"), + parentHash: null, + proposerAddress: "PROPOSER", + transactions: [ + { + index: 0, + hash: Buffer.from("bb".repeat(32), "hex"), + code: 0, + gasUsed: 0, + gasWanted: 0, + fee: [], + messages: typeUrls.map((typeUrl, index) => ({ index, typeUrl, body: null })) + } + ] + }; + } +}); diff --git a/apps/chain-indexer/src/pipeline/block-committer.service.ts b/apps/chain-indexer/src/pipeline/block-committer.service.ts index a730080b38..056c467cf1 100644 --- a/apps/chain-indexer/src/pipeline/block-committer.service.ts +++ b/apps/chain-indexer/src/pipeline/block-committer.service.ts @@ -73,18 +73,38 @@ export class BlockCommitterService { async #internMessageTypes(block: DecodedBlock): Promise> { const typeUrls = new Set(block.transactions.flatMap(tx => tx.messages.map(message => message.typeUrl))); - const missing = [...typeUrls].filter(typeUrl => !this.#typeIds.has(typeUrl)); + const uncached = [...typeUrls].filter(typeUrl => !this.#typeIds.has(typeUrl)); - if (missing.length > 0) { - await this.#db - .insert(MessageTypes) - .values(missing.map(type => ({ type }))) - .onConflictDoNothing(); - - const rows = await this.#db.select().from(MessageTypes).where(inArray(MessageTypes.type, missing)); - rows.forEach(row => this.#typeIds.set(row.type, row.id)); + if (uncached.length > 0) { + await this.#cacheTypeIds(uncached); } return this.#typeIds; } + + /** Existing rows are selected before inserting: an insert that conflicts still consumes the id sequence, which would exhaust it across restarts. */ + async #cacheTypeIds(uncached: string[]): Promise { + const existing = await this.#db.select().from(MessageTypes).where(inArray(MessageTypes.type, uncached)); + existing.forEach(row => this.#typeIds.set(row.type, row.id)); + + const missing = uncached.filter(typeUrl => !this.#typeIds.has(typeUrl)); + + if (missing.length === 0) { + return; + } + + const inserted = await this.#db + .insert(MessageTypes) + .values(missing.map(type => ({ type }))) + .onConflictDoNothing() + .returning(); + inserted.forEach(row => this.#typeIds.set(row.type, row.id)); + + const insertedConcurrently = missing.filter(typeUrl => !this.#typeIds.has(typeUrl)); + + if (insertedConcurrently.length > 0) { + const rows = await this.#db.select().from(MessageTypes).where(inArray(MessageTypes.type, insertedConcurrently)); + rows.forEach(row => this.#typeIds.set(row.type, row.id)); + } + } } diff --git a/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts b/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts index ed144eea6b..ede66dbc3b 100644 --- a/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts +++ b/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts @@ -63,6 +63,15 @@ describe(BlockDecoderService.name, () => { expect(decoded.transactions[0].messages[0].body).toBeNull(); }); + it("throws when the tx results count does not match the block txs", () => { + const { decoder } = setup(); + const rawTx = buildMsgSendTx(); + + expect(() => decoder.decode(buildBlock({ txs: [rawTx.toString("base64")] }), { height: "1234", txs_results: null })).toThrow( + "Block 1234 has 1 txs but 0 tx results" + ); + }); + it("marks failed transactions with their error code", () => { const { decoder } = setup(); const rawTx = buildMsgSendTx(); diff --git a/apps/chain-indexer/src/pipeline/block-decoder.service.ts b/apps/chain-indexer/src/pipeline/block-decoder.service.ts index 01e7b0b258..987f68b935 100644 --- a/apps/chain-indexer/src/pipeline/block-decoder.service.ts +++ b/apps/chain-indexer/src/pipeline/block-decoder.service.ts @@ -25,6 +25,10 @@ export class BlockDecoderService { const rawTxs = block.block.data.txs; const txResults = blockResults.txs_results ?? []; + if (rawTxs.length !== txResults.length) { + throw new Error(`Block ${block.block.header.height} has ${rawTxs.length} txs but ${txResults.length} tx results`); + } + return { height: parseInt(block.block.header.height), datetime: new Date(block.block.header.time), @@ -35,16 +39,16 @@ export class BlockDecoderService { }; } - #decodeTransaction(rawTxBase64: string, txResult: RpcTxResult | undefined, index: number): DecodedTransaction { + #decodeTransaction(rawTxBase64: string, txResult: RpcTxResult, index: number): DecodedTransaction { const rawTx = fromBase64(rawTxBase64); const decodedTx = decodeTxRaw(rawTx); return { index, hash: createHash("sha256").update(rawTx).digest(), - code: txResult?.code ?? 0, - gasUsed: parseInt(txResult?.gas_used ?? "0"), - gasWanted: parseInt(txResult?.gas_wanted ?? "0"), + code: txResult.code ?? 0, + gasUsed: parseInt(txResult.gas_used ?? "0"), + gasWanted: parseInt(txResult.gas_wanted ?? "0"), fee: decodedTx.authInfo.fee?.amount.map(({ denom, amount }) => ({ denom, amount })) ?? [], messages: decodedTx.body.messages.map((message, messageIndex) => this.#decodeMessage(message, messageIndex)) }; diff --git a/apps/chain-indexer/src/pipeline/canonical-json.spec.ts b/apps/chain-indexer/src/pipeline/canonical-json.spec.ts index f9d7476ea2..0996c42558 100644 --- a/apps/chain-indexer/src/pipeline/canonical-json.spec.ts +++ b/apps/chain-indexer/src/pipeline/canonical-json.spec.ts @@ -15,6 +15,10 @@ describe(toCanonicalJson.name, () => { expect(toCanonicalJson({ payload: Uint8Array.from([1, 2, 3]) }, 1_000)).toEqual({ payload: Buffer.from([1, 2, 3]).toString("base64") }); }); + it("converts Buffers to base64 strings instead of Buffer#toJSON output", () => { + expect(toCanonicalJson({ payload: Buffer.from([1, 2, 3]) }, 1_000)).toEqual({ payload: Buffer.from([1, 2, 3]).toString("base64") }); + }); + it("returns null when the serialized value exceeds maxBytes", () => { expect(toCanonicalJson({ memo: "x".repeat(100) }, 50)).toBeNull(); }); diff --git a/apps/chain-indexer/src/pipeline/canonical-json.ts b/apps/chain-indexer/src/pipeline/canonical-json.ts index cce85c9f01..90c506f6bb 100644 --- a/apps/chain-indexer/src/pipeline/canonical-json.ts +++ b/apps/chain-indexer/src/pipeline/canonical-json.ts @@ -1,9 +1,12 @@ -function replacer(this: unknown, key: string, value: unknown): unknown { - if (typeof value === "bigint") { - return value.toString(); +/** JSON.stringify calls toJSON before the replacer, so the pre-toJSON value is read from the holder (`this`) to serialize Buffers as base64 instead of Buffer#toJSON output. */ +function replacer(this: Record, key: string, value: unknown): unknown { + const original = this[key]; + + if (typeof original === "bigint") { + return original.toString(); } - if (value instanceof Uint8Array) { - return Buffer.from(value).toString("base64"); + if (original instanceof Uint8Array) { + return Buffer.from(original).toString("base64"); } return value; } diff --git a/apps/chain-indexer/src/pipeline/sync-runner.service.ts b/apps/chain-indexer/src/pipeline/sync-runner.service.ts index acdbc27074..16350f8113 100644 --- a/apps/chain-indexer/src/pipeline/sync-runner.service.ts +++ b/apps/chain-indexer/src/pipeline/sync-runner.service.ts @@ -19,6 +19,16 @@ import { RpcClientPool } from "@src/rpc/rpc-client-pool.service"; const SYNC_LEADER_LOCK_KEY = 7_431_001; const LEADERSHIP_RETRY_MS = 5_000; const PROGRESS_LOG_EVERY_BLOCKS = 100; +const LEADERSHIP_CHECK_EVERY_BLOCKS = 100; +const TRANSIENT_RETRY_MAX_ATTEMPTS = 10; +const TRANSIENT_RETRY_BASE_MS = 1_000; +const TRANSIENT_RETRY_MAX_MS = 30_000; + +/** Parent-hash continuity break; fatal by design so the process halts instead of committing a forked history. */ +export class ChainContinuityError extends Error {} + +/** The advisory-lock session was replaced (e.g. a transparent driver reconnect), so another process may hold leadership. */ +export class SyncLeadershipLostError extends Error {} @singleton() export class SyncRunnerService { @@ -32,6 +42,7 @@ export class SyncRunnerService { #stopped = false; #reserved: postgres.ReservedSql | undefined; + #leaderBackendPid: number | undefined; #lastHash: Buffer | null = null; constructor( @@ -67,10 +78,15 @@ export class SyncRunnerService { async dispose(): Promise { this.#stopped = true; + this.#releaseReservedLockConnection(); + } + + /** release() may throw when the pg pool was ended first; the connection is already gone then, which is the goal. */ + #releaseReservedLockConnection(): void { try { this.#reserved?.release(); - } catch { - /** the reserved connection may already be gone when the pool was ended first */ + } catch (error) { + this.#logger.debug({ event: "SYNC_LOCK_RELEASE_SKIPPED", error }); } } @@ -80,7 +96,8 @@ export class SyncRunnerService { this.#logger.info({ event: "SYNC_STARTED", network: this.#config.NETWORK, nextHeight }); while (!this.#stopped) { - const tipHeight = await this.#getTipHeight(); + await this.#assertLeadership(); + const tipHeight = await this.#retryTransient(() => this.#getTipHeight(), { event: "SYNC_TIP_FETCH_RETRY" }); if (nextHeight > tipHeight) { await delay(this.#config.SYNC_POLL_INTERVAL_MS); @@ -88,8 +105,35 @@ export class SyncRunnerService { } while (nextHeight <= tipHeight && !this.#stopped) { - await this.#syncBlock(nextHeight); + const height = nextHeight; + await this.#retryTransient(() => this.#syncBlock(height), { event: "SYNC_BLOCK_RETRY", height }); nextHeight++; + + if (nextHeight % LEADERSHIP_CHECK_EVERY_BLOCKS === 0) { + await this.#assertLeadership(); + } + } + } + } + + /** Transient failures (RPC timeouts, connection drops) are retried with capped exponential backoff; persistent ones and fatal errors propagate and halt the process. */ + async #retryTransient(operation: () => Promise, logContext: { event: string; height?: number }): Promise { + let attempt = 0; + + while (true) { + attempt++; + try { + return await operation(); + } catch (error) { + const isFatal = error instanceof ChainContinuityError || error instanceof SyncLeadershipLostError; + + if (this.#stopped || isFatal || attempt >= TRANSIENT_RETRY_MAX_ATTEMPTS) { + throw error; + } + + const delayMs = Math.min(TRANSIENT_RETRY_BASE_MS * 2 ** (attempt - 1), TRANSIENT_RETRY_MAX_MS); + this.#logger.warn({ ...logContext, attempt, delayMs, error }); + await delay(delayMs); } } } @@ -117,7 +161,7 @@ export class SyncRunnerService { expectedParentHash: this.#lastHash.toString("hex"), actualParentHash: block.parentHash.toString("hex") }); - throw new Error(`Parent hash mismatch at height ${block.height}; halting sync`); + throw new ChainContinuityError(`Parent hash mismatch at height ${block.height}; halting sync`); } } @@ -128,7 +172,9 @@ export class SyncRunnerService { const [{ acquired }] = await this.#reserved`SELECT pg_try_advisory_lock(${SYNC_LEADER_LOCK_KEY}) AS acquired`; if (acquired) { - this.#logger.info({ event: "SYNC_LEADERSHIP_ACQUIRED" }); + const [{ pid }] = await this.#reserved`SELECT pg_backend_pid() AS pid`; + this.#leaderBackendPid = pid; + this.#logger.info({ event: "SYNC_LEADERSHIP_ACQUIRED", backendPid: pid }); return; } @@ -137,6 +183,20 @@ export class SyncRunnerService { } } + /** Advisory locks are session-scoped: a transparent driver reconnect creates a fresh session WITHOUT the lock, so the backend pid is re-checked to detect silent leadership loss before more blocks are committed. */ + async #assertLeadership(): Promise { + if (!this.#reserved) { + throw new SyncLeadershipLostError("Reserved advisory-lock connection is gone; halting sync"); + } + + const [{ pid }] = await this.#reserved`SELECT pg_backend_pid() AS pid`; + + if (pid !== this.#leaderBackendPid) { + this.#logger.error({ event: "SYNC_LEADERSHIP_LOST", expectedBackendPid: this.#leaderBackendPid, actualBackendPid: pid }); + throw new SyncLeadershipLostError(`Advisory-lock session changed (backend pid ${this.#leaderBackendPid} -> ${pid}); halting sync`); + } + } + async #resolveStartHeight(): Promise { const [state] = await this.#db.select().from(IndexerState).where(eq(IndexerState.stream, SYNC_STREAM)); diff --git a/apps/chain-indexer/src/providers/db.provider.ts b/apps/chain-indexer/src/providers/db.provider.ts index 90e837c5a7..b62480732b 100644 --- a/apps/chain-indexer/src/providers/db.provider.ts +++ b/apps/chain-indexer/src/providers/db.provider.ts @@ -17,12 +17,16 @@ container.register(CHAIN_DB, { useFactory: instancePerContainerCachingFactory(createDatabase) }); +/** Arbitrary but fixed application-wide key serializing migrateDb() across concurrently starting processes; drizzle's migrator has no locking of its own. */ +const MIGRATION_LOCK_KEY = 7_431_000; + export async function migrateDb(): Promise { const config = container.resolve(APP_CONFIG); const migrationClient = postgres(config.POSTGRES_DB_URI, { max: 1 }); const migrationDatabase = drizzle(migrationClient, { schema }); try { + await migrationClient`SELECT pg_advisory_lock(${MIGRATION_LOCK_KEY})`; await migrate(migrationDatabase, { migrationsFolder: config.DRIZZLE_MIGRATIONS_FOLDER }); } finally { await migrationClient.end(); diff --git a/apps/chain-indexer/src/providers/type-registry.provider.ts b/apps/chain-indexer/src/providers/type-registry.provider.ts index 7e43ec3804..84cf7d2c1d 100644 --- a/apps/chain-indexer/src/providers/type-registry.provider.ts +++ b/apps/chain-indexer/src/providers/type-registry.provider.ts @@ -9,7 +9,7 @@ import type { GeneratedType } from "@cosmjs/proto-signing"; import { Registry } from "@cosmjs/proto-signing"; import { defaultRegistryTypes as stargateDefaultRegistryTypes } from "@cosmjs/stargate"; import type { InjectionToken } from "tsyringe"; -import { container, inject } from "tsyringe"; +import { container } from "tsyringe"; type DefaultRegistryType = [string, GeneratedType]; const ibcTypes: DefaultRegistryType[] = stargateDefaultRegistryTypes.filter(([type]) => type.startsWith("/ibc")); @@ -33,4 +33,3 @@ export const TYPE_REGISTRY: InjectionToken = Symbol("TYPE_REGISTRY"); export type { Registry }; container.register(TYPE_REGISTRY, { useValue: registry }); -export const InjectTypeRegistry = (): ReturnType => inject(TYPE_REGISTRY); diff --git a/apps/chain-indexer/src/rpc/rpc-types.ts b/apps/chain-indexer/src/rpc/rpc-types.ts index 54d3f4969c..fec8502dbd 100644 --- a/apps/chain-indexer/src/rpc/rpc-types.ts +++ b/apps/chain-indexer/src/rpc/rpc-types.ts @@ -23,11 +23,12 @@ export interface RpcBlockResult { }; } +/** Fields marshaled with proto3 omitempty semantics may be absent when zero (e.g. code 0 on success). */ export interface RpcTxResult { - code: number; + code?: number; log?: string; - gas_used: string; - gas_wanted: string; + gas_used?: string; + gas_wanted?: string; } export interface RpcBlockResultsResult { diff --git a/apps/chain-indexer/src/server.ts b/apps/chain-indexer/src/server.ts index f64e65c9d3..9bcffbcddf 100644 --- a/apps/chain-indexer/src/server.ts +++ b/apps/chain-indexer/src/server.ts @@ -1,6 +1,8 @@ +import { createOtelLogger } from "@akashnetwork/logging/otel"; + import { bootstrap } from "./index"; void bootstrap().catch(error => { - console.error("Failed to bootstrap chain-indexer", error); + createOtelLogger({ context: "APP" }).error({ event: "BOOTSTRAP_FAILED", error }); process.exitCode = 1; }); diff --git a/apps/chain-indexer/src/services/hono-error-handler/hono-error-handler.service.ts b/apps/chain-indexer/src/services/hono-error-handler/hono-error-handler.service.ts index 4654e46aee..e074feddc0 100644 --- a/apps/chain-indexer/src/services/hono-error-handler/hono-error-handler.service.ts +++ b/apps/chain-indexer/src/services/hono-error-handler/hono-error-handler.service.ts @@ -56,11 +56,10 @@ export class HonoErrorHandlerService { ); } - const message = error instanceof Error ? error.message : "Internal server error"; return c.json( { error: "InternalServerError", - message, + message: "Internal server error", code: "internal_server_error", type: "server_error" }, diff --git a/apps/chain-indexer/src/services/shutdown-server/shutdown-server.ts b/apps/chain-indexer/src/services/shutdown-server/shutdown-server.ts index 0b62a13cf7..3d6dad520b 100644 --- a/apps/chain-indexer/src/services/shutdown-server/shutdown-server.ts +++ b/apps/chain-indexer/src/services/shutdown-server/shutdown-server.ts @@ -1,6 +1,9 @@ import type { Logger } from "@akashnetwork/logging"; import type { ServerType } from "@hono/node-server"; +/** Grace period for in-flight requests before remaining keep-alive sockets are destroyed so close() can complete. */ +const FORCE_CLOSE_GRACE_MS = 10_000; + export async function shutdownServer(server: ServerType, appLogger: Logger, onShutdown?: () => void | Promise): Promise { return new Promise(resolve => { const shutdown = (error?: unknown) => { @@ -21,6 +24,7 @@ export async function shutdownServer(server: ServerType, appLogger: Logger, onSh try { if (server.listening) { server.close(shutdown); + closeConnections(server); } else { shutdown(); } @@ -29,3 +33,13 @@ export async function shutdownServer(server: ServerType, appLogger: Logger, onSh } }); } + +/** close() only stops new connections: idle keep-alive sockets are closed immediately and stragglers destroyed after a grace period so close() can ever finish. */ +function closeConnections(server: ServerType): void { + if (!("closeIdleConnections" in server)) { + return; + } + + server.closeIdleConnections(); + setTimeout(() => server.closeAllConnections(), FORCE_CLOSE_GRACE_MS).unref(); +} diff --git a/apps/chain-indexer/src/services/start-server/start-server.ts b/apps/chain-indexer/src/services/start-server/start-server.ts index 0fc9e6b29e..83efc067f4 100644 --- a/apps/chain-indexer/src/services/start-server/start-server.ts +++ b/apps/chain-indexer/src/services/start-server/start-server.ts @@ -18,7 +18,7 @@ export async function startServer( beforeStart?: () => Promise; container?: DependencyContainer; } -): Promise { +): Promise { const container = options.container ?? rootContainer; const disposeContainerOnce = once(() => { logger.info({ event: "DISPOSING_CONTAINER" }); @@ -36,22 +36,47 @@ export async function startServer( await disposeContainerOnce(); } }); + try { await options.beforeStart?.(); logger.info({ event: "SERVER_STARTING", url: `http://localhost:${options.port}`, NODE_OPTIONS: process.env.NODE_OPTIONS }); - server = serve({ + const startedServer = serve({ fetch: app.fetch, port: options.port }); + server = startedServer; + await waitUntilListening(startedServer); - server.on("close", disposeContainerOnce); + startedServer.on("error", error => { + logger.error({ event: "SERVER_ERROR", error }); + void shutdown("SERVER_ERROR"); + }); + startedServer.on("close", disposeContainerOnce); processEvents.on("SIGTERM", () => shutdown("SIGTERM")); processEvents.on("SIGINT", () => shutdown("SIGINT")); - processEvents.on("exit", exitCode => shutdown(`EXIT:${exitCode}`)); - return server; + + return startedServer; } catch (error) { logger.error({ event: "SERVER_START_ERROR", error }); await shutdown("SERVER_START_ERROR"); + throw error; } } + +/** serve() binds asynchronously: failures like EADDRINUSE surface as an "error" event, never as a synchronous throw. */ +function waitUntilListening(server: ServerType): Promise { + return new Promise((resolve, reject) => { + if (server.listening) { + resolve(); + return; + } + + const settleOnError = (error: unknown) => reject(error); + server.once("error", settleOnError); + server.once("listening", () => { + server.off("error", settleOnError); + resolve(); + }); + }); +} diff --git a/apps/chain-indexer/test/setup-unit-tests.ts b/apps/chain-indexer/test/setup-unit-tests.ts index 058eb16608..d5754ad867 100644 --- a/apps/chain-indexer/test/setup-unit-tests.ts +++ b/apps/chain-indexer/test/setup-unit-tests.ts @@ -2,9 +2,5 @@ import { container } from "tsyringe"; import { afterAll } from "vitest"; afterAll(async () => { - try { - await container.dispose(); - } catch { - /** container may already be disposed in tests */ - } + await container.dispose(); }); From 6a0a33575d1a39e6a9e4c0e3e113f3368194dbbc Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Wed, 12 Aug 2026 11:29:50 +0530 Subject: [PATCH 03/14] feat(indexer): backfill role for historical catch-up (#3582) * feat(indexer): add backfill range and tuning env config * feat(indexer): support ordered batched commits with a parameterized checkpoint stream * feat(indexer): add backfill range planner * feat(indexer): implement the backfill role for historical catch-up * fix(indexer): address backfill review findings - attach a no-op catch to prefetched blocks so a rejection settling before its height is consumed cannot crash as an unhandled rejection - check range completion before tip validity so a finished range stays a no-op even when a lagging node reports a stale tip - extract the shared PgAdvisoryLeaderLock used by both runners - use lodash chunk instead of a local helper - build the pg client test double with a nested mock instead of a cast * refactor(indexer): share retry backoff and runner-role lifecycle across roles - extract retryWithBackoff and use it in both runners - retry backfill startup reads and idempotent batch commits so a transient blip no longer kills a multi-hour job - collapse the sync and backfill bootstrap cases into one helper * refactor(indexer): move continuity error and transient retry policy to shared modules * refactor(indexer): retry the continuity seed read like other transient reads * refactor(indexer): drop advisory-lock leader election for concurrency-safe writes Inserts were already idempotent; making the indexer_state checkpoint upsert monotonic (GREATEST) makes overlapping writers harmless, so the leader lock, its backend-pid liveness checks, and LeadershipLostError can all go. Single writers per role are enforced at the orchestration level instead. * refactor(indexer): use the exported LoggerService type instead of a local alias * fix(indexer): retry an interrupted backfill instead of completing the Job Addresses review findings on the backfill role: - A backfill stopped before finishing its range (SIGTERM mid-run) now throws RunnerInterruptedError so the process exits non-zero. Backfill runs as a K8s Job where exit 0 marks it Complete, so the previous clean exit silently left the range unfinished and unretried; it now resumes from its checkpoint on the next attempt. - Drop the unreachable prefetch fallback in the consume loop; the sliding window always has the current height inflight before it is awaited. - Back the retry delay with the global setTimeout so the fetch-retry test can advance it with fake timers instead of sleeping a real second. * fix(indexer): surface env validation errors without the DI wrapper A misconfigured role (e.g. a backfill Job missing BACKFILL_FROM/TO_HEIGHT) failed with a tsyringe "Cannot inject the dependency" wrapper around the ZodError plus an internal stack trace. Validate env eagerly at bootstrap and log a CONFIG_INVALID event listing the offending fields and messages instead. * refactor(indexer): parallelize backfill startup reads and signal completion via return Fetch the checkpoint height and chain tip concurrently instead of serially, hoist the loop-invariant base height in the contiguity check, and have the backfill run return whether it completed so start() no longer re-reads the stopped flag. * fix(indexer): track backfill completion by committed height, not the stopped flag A shutdown landing during the final commit left the range fully committed but reported it as interrupted, failing the Job and triggering a spurious retry. Completion is now derived from the last committed height reaching the range end. Also dedupes the identical getTipHeight helper into RpcClientPool so both runners share one implementation. --- apps/chain-indexer/README.md | 16 +- apps/chain-indexer/env/.env.sample | 4 + .../src/config/env.config.spec.ts | 28 ++ apps/chain-indexer/src/config/env.config.ts | 26 +- apps/chain-indexer/src/index.ts | 70 ++++- .../retry-with-backoff.spec.ts | 38 +++ .../retry-with-backoff/retry-with-backoff.ts | 35 +++ .../src/pipeline/backfill-planner.spec.ts | 41 +++ .../src/pipeline/backfill-planner.ts | 29 ++ .../pipeline/backfill-runner.service.spec.ts | 254 ++++++++++++++++++ .../src/pipeline/backfill-runner.service.ts | 241 +++++++++++++++++ .../pipeline/block-committer.service.spec.ts | 72 ++++- .../src/pipeline/block-committer.service.ts | 109 +++++--- .../src/pipeline/chain-continuity-error.ts | 2 + .../src/pipeline/runner-interrupted-error.ts | 6 + .../src/pipeline/sync-runner.service.ts | 99 +------ .../src/pipeline/transient-retry.ts | 21 ++ .../src/rpc/rpc-client-pool.service.spec.ts | 7 + .../src/rpc/rpc-client-pool.service.ts | 5 + 19 files changed, 954 insertions(+), 149 deletions(-) create mode 100644 apps/chain-indexer/src/lib/retry-with-backoff/retry-with-backoff.spec.ts create mode 100644 apps/chain-indexer/src/lib/retry-with-backoff/retry-with-backoff.ts create mode 100644 apps/chain-indexer/src/pipeline/backfill-planner.spec.ts create mode 100644 apps/chain-indexer/src/pipeline/backfill-planner.ts create mode 100644 apps/chain-indexer/src/pipeline/backfill-runner.service.spec.ts create mode 100644 apps/chain-indexer/src/pipeline/backfill-runner.service.ts create mode 100644 apps/chain-indexer/src/pipeline/chain-continuity-error.ts create mode 100644 apps/chain-indexer/src/pipeline/runner-interrupted-error.ts create mode 100644 apps/chain-indexer/src/pipeline/transient-retry.ts diff --git a/apps/chain-indexer/README.md b/apps/chain-indexer/README.md index a98f36f8f3..14750241c0 100644 --- a/apps/chain-indexer/README.md +++ b/apps/chain-indexer/README.md @@ -9,7 +9,9 @@ INDEXER_ROLE = sync | backfill | api | jobs NETWORK = mainnet | sandbox | testnet ``` -Currently implemented: `sync` (live tail with per-block atomic commits, advisory-lock leader election, parent-hash continuity check) and a minimal `api` (healthz + status). `backfill` and `jobs` exit with `ROLE_NOT_IMPLEMENTED`. +Currently implemented: `sync` (live tail with per-block atomic commits and a parent-hash continuity check), `backfill` (historical catch-up over an explicit height range), and a minimal `api` (healthz + status). `jobs` exits with `ROLE_NOT_IMPLEMENTED`. + +Writers do not use leader election. Inserts are natural-keyed and conflict-ignoring and the `indexer_state` checkpoint only moves forward (`GREATEST` upsert), so overlapping writers on the same stream (e.g. two pods during a rolling deploy) duplicate work but cannot corrupt data or regress the checkpoint. Run one replica per writer role (`replicas: 1` for sync, `parallelism: 1` for backfill Jobs) to avoid the wasted work. ## Running locally @@ -34,6 +36,18 @@ curl localhost:3092/v1/status The checkpoint height should advance as blocks land in `cosmos.blocks`, `cosmos.transactions`, and `cosmos.messages`. +## Backfill + +The backfill role fills the database over an explicit, inclusive height range and exits when done, so it fits a one-off K8s Job: + +``` +INDEXER_ROLE=backfill +BACKFILL_FROM_HEIGHT=100000 +BACKFILL_TO_HEIGHT=200000 +``` + +Blocks are fetched from RPC in parallel (`BACKFILL_CONCURRENCY`, default 10) and committed strictly in order in batches of `BACKFILL_BATCH_SIZE` blocks (default 200), each batch in one Postgres transaction together with the checkpoint advance. Progress is checkpointed per range under the `indexer_state` stream `backfill:{from}-{to}`, so killing and restarting the job resumes at the checkpoint without gaps or duplicates, and re-running a completed range exits 0 immediately. Changing the range creates a fresh checkpoint row. All inserts are natural-keyed and conflict-ignoring, so a backfill can run against the same database as live sync, and a duplicate backfill pod on the same range is harmless. + ## Tests ```bash diff --git a/apps/chain-indexer/env/.env.sample b/apps/chain-indexer/env/.env.sample index ec4a54572a..78363d5847 100644 --- a/apps/chain-indexer/env/.env.sample +++ b/apps/chain-indexer/env/.env.sample @@ -4,6 +4,10 @@ POSTGRES_DB_URI=postgres://user:password@localhost:5432/chain-indexer RPC_NODE_ENDPOINTS= SYNC_START_HEIGHT= SYNC_POLL_INTERVAL_MS=3000 +BACKFILL_FROM_HEIGHT= +BACKFILL_TO_HEIGHT= +BACKFILL_CONCURRENCY=10 +BACKFILL_BATCH_SIZE=200 RPC_TIMEOUT_MS=15000 MESSAGE_BODY_MAX_BYTES=65536 PORT=3092 diff --git a/apps/chain-indexer/src/config/env.config.spec.ts b/apps/chain-indexer/src/config/env.config.spec.ts index 5bcf5efab2..cbafcbfc98 100644 --- a/apps/chain-indexer/src/config/env.config.spec.ts +++ b/apps/chain-indexer/src/config/env.config.spec.ts @@ -31,6 +31,34 @@ describe("envSchema", () => { expect(() => setup({ PORT: "3092.5" })).toThrow(); }); + describe("when INDEXER_ROLE is backfill", () => { + it("requires both backfill heights", () => { + expect(() => setup({ INDEXER_ROLE: "backfill" })).toThrow(/BACKFILL_FROM_HEIGHT[\s\S]*BACKFILL_TO_HEIGHT/); + }); + + it("rejects a range where from is above to", () => { + expect(() => setup({ INDEXER_ROLE: "backfill", BACKFILL_FROM_HEIGHT: "100", BACKFILL_TO_HEIGHT: "50" })).toThrow( + "BACKFILL_FROM_HEIGHT must be <= BACKFILL_TO_HEIGHT" + ); + }); + + it("parses a valid range with concurrency and batch size defaults", () => { + const config = setup({ INDEXER_ROLE: "backfill", BACKFILL_FROM_HEIGHT: "100", BACKFILL_TO_HEIGHT: "200" }); + + expect(config.BACKFILL_FROM_HEIGHT).toBe(100); + expect(config.BACKFILL_TO_HEIGHT).toBe(200); + expect(config.BACKFILL_CONCURRENCY).toBe(10); + expect(config.BACKFILL_BATCH_SIZE).toBe(200); + }); + }); + + it("does not require backfill heights for other roles", () => { + const config = setup({ INDEXER_ROLE: "api", BACKFILL_FROM_HEIGHT: "", BACKFILL_TO_HEIGHT: "" }); + + expect(config.BACKFILL_FROM_HEIGHT).toBeUndefined(); + expect(config.BACKFILL_TO_HEIGHT).toBeUndefined(); + }); + function setup(overrides?: Record) { return envSchema.parse({ POSTGRES_DB_URI: "postgres://unit:unit@localhost:5432/unit", ...overrides }); } diff --git a/apps/chain-indexer/src/config/env.config.ts b/apps/chain-indexer/src/config/env.config.ts index eb2ec6e76d..1c00840be0 100644 --- a/apps/chain-indexer/src/config/env.config.ts +++ b/apps/chain-indexer/src/config/env.config.ts @@ -3,7 +3,7 @@ import { z } from "zod"; /** Treats an empty string as absent so `VAR=` lines in env files don't fail coerced-number validation. */ const emptyStringAsUndefined = (value: unknown) => (value === "" ? undefined : value); -export const envSchema = z.object({ +const rawEnvSchema = z.object({ INDEXER_ROLE: z.enum(["sync", "backfill", "api", "jobs"]).default("sync"), NETWORK: z.enum(["mainnet", "sandbox", "testnet"]).default("sandbox"), POSTGRES_DB_URI: z.string(), @@ -14,6 +14,14 @@ export const envSchema = z.object({ /** First height to sync when the database has no checkpoint yet. Defaults to the current chain tip. */ SYNC_START_HEIGHT: z.preprocess(emptyStringAsUndefined, z.number({ coerce: true }).int().positive().optional()), SYNC_POLL_INTERVAL_MS: z.number({ coerce: true }).int().positive().default(3_000), + /** First height of the backfill range (inclusive). Required when INDEXER_ROLE is "backfill". */ + BACKFILL_FROM_HEIGHT: z.preprocess(emptyStringAsUndefined, z.number({ coerce: true }).int().positive().optional()), + /** Last height of the backfill range (inclusive). Required when INDEXER_ROLE is "backfill". */ + BACKFILL_TO_HEIGHT: z.preprocess(emptyStringAsUndefined, z.number({ coerce: true }).int().positive().optional()), + /** How many blocks the backfill fetches from RPC in parallel. */ + BACKFILL_CONCURRENCY: z.number({ coerce: true }).int().min(1).max(64).default(10), + /** How many blocks the backfill commits per Postgres transaction. */ + BACKFILL_BATCH_SIZE: z.number({ coerce: true }).int().min(1).max(1_000).default(200), /** Decoded message bodies above this serialized size are stored as null to keep pathological messages out of Postgres. */ MESSAGE_BODY_MAX_BYTES: z.number({ coerce: true }).int().positive().default(65_536), DRIZZLE_MIGRATIONS_FOLDER: z.string().default("./drizzle"), @@ -23,4 +31,20 @@ export const envSchema = z.object({ PORT: z.number({ coerce: true }).int().min(1).max(65_535).optional().default(3092) }); +export const envSchema = rawEnvSchema.superRefine((env, ctx) => { + if (env.INDEXER_ROLE !== "backfill") { + return; + } + + (["BACKFILL_FROM_HEIGHT", "BACKFILL_TO_HEIGHT"] as const).forEach(key => { + if (env[key] === undefined) { + ctx.addIssue({ code: z.ZodIssueCode.custom, path: [key], message: 'Required when INDEXER_ROLE is "backfill"' }); + } + }); + + if (env.BACKFILL_FROM_HEIGHT !== undefined && env.BACKFILL_TO_HEIGHT !== undefined && env.BACKFILL_FROM_HEIGHT > env.BACKFILL_TO_HEIGHT) { + ctx.addIssue({ code: z.ZodIssueCode.custom, path: ["BACKFILL_FROM_HEIGHT"], message: "BACKFILL_FROM_HEIGHT must be <= BACKFILL_TO_HEIGHT" }); + } +}); + export type EnvConfig = z.infer; diff --git a/apps/chain-indexer/src/index.ts b/apps/chain-indexer/src/index.ts index 3944fc6b74..db288b48b7 100644 --- a/apps/chain-indexer/src/index.ts +++ b/apps/chain-indexer/src/index.ts @@ -1,10 +1,14 @@ import "reflect-metadata"; import "@src/providers"; +import type { LoggerService } from "@akashnetwork/logging"; import { createOtelLogger } from "@akashnetwork/logging/otel"; import { container } from "tsyringe"; import { createApp } from "@src/app"; +import { envSchema } from "@src/config/env.config"; +import { BackfillRunnerService } from "@src/pipeline/backfill-runner.service"; +import { RunnerInterruptedError } from "@src/pipeline/runner-interrupted-error"; import { SyncRunnerService } from "@src/pipeline/sync-runner.service"; import { migrateDb } from "@src/providers/db.provider"; import { AppConfigService } from "@src/services/app-config/app-config.service"; @@ -12,28 +16,29 @@ import { shutdownServer } from "@src/services/shutdown-server/shutdown-server"; import { startServer } from "@src/services/start-server/start-server"; export async function bootstrap(): Promise { + const logger = createOtelLogger({ context: "APP" }); + + if (!validateConfig(logger)) { + process.exitCode = 1; + return; + } + const config = container.resolve(AppConfigService); const role = config.get("INDEXER_ROLE"); - const logger = createOtelLogger({ context: "APP" }); + const port = config.get("PORT"); switch (role) { case "sync": { - await migrateDb(); - const server = await startServer(createApp(), logger, process, { port: config.get("PORT") }); - - try { - await container.resolve(SyncRunnerService).start(); - } catch (error) { - logger.error({ event: "SYNC_FATAL", error }); - process.exitCode = 1; - } - - await shutdownServer(server, logger); + await runRunnerBehindServer(() => container.resolve(SyncRunnerService), "SYNC_FATAL", logger, port); + return; + } + case "backfill": { + await runRunnerBehindServer(() => container.resolve(BackfillRunnerService), "BACKFILL_FATAL", logger, port); return; } case "api": { await migrateDb(); - await startServer(createApp(), logger, process, { port: config.get("PORT") }); + await startServer(createApp(), logger, process, { port }); return; } default: { @@ -42,3 +47,42 @@ export async function bootstrap(): Promise { } } } + +/** Validates env eagerly so a misconfigured role (e.g. a backfill Job missing BACKFILL_FROM/TO_HEIGHT) fails with the actual field errors instead of a tsyringe dependency-injection wrapper around the ZodError. */ +function validateConfig(logger: LoggerService): boolean { + const result = envSchema.safeParse(process.env); + + if (result.success) { + return true; + } + + logger.error({ + event: "CONFIG_INVALID", + issues: result.error.issues.map(issue => ({ path: issue.path.join(".") || "(root)", message: issue.message })) + }); + return false; +} + +/** + * Shared runner-role lifecycle: migrate, serve healthz, run to completion, then shut the server + * down so the process can exit. A fatal error exits non-zero; a run stopped before finishing + * (`RunnerInterruptedError`, e.g. SIGTERM mid-backfill) also exits non-zero so a K8s Job is retried + * and resumes from its checkpoint rather than being marked Complete with the range unfinished. + */ +async function runRunnerBehindServer(resolveRunner: () => { start(): Promise }, fatalEvent: string, logger: LoggerService, port: number): Promise { + await migrateDb(); + const server = await startServer(createApp(), logger, process, { port }); + + try { + await resolveRunner().start(); + } catch (error) { + if (error instanceof RunnerInterruptedError) { + logger.warn({ event: "RUNNER_INTERRUPTED", error }); + } else { + logger.error({ event: fatalEvent, error }); + } + process.exitCode = 1; + } + + await shutdownServer(server, logger); +} diff --git a/apps/chain-indexer/src/lib/retry-with-backoff/retry-with-backoff.spec.ts b/apps/chain-indexer/src/lib/retry-with-backoff/retry-with-backoff.spec.ts new file mode 100644 index 0000000000..6b4589d319 --- /dev/null +++ b/apps/chain-indexer/src/lib/retry-with-backoff/retry-with-backoff.spec.ts @@ -0,0 +1,38 @@ +import { describe, expect, it, vi } from "vitest"; + +import { retryWithBackoff } from "@src/lib/retry-with-backoff/retry-with-backoff"; + +describe(retryWithBackoff.name, () => { + it("returns the result after a transient failure", async () => { + const operation = vi.fn().mockRejectedValueOnce(new Error("transient")).mockResolvedValue("ok"); + const onRetry = vi.fn(); + + const result = await retryWithBackoff(operation, { maxAttempts: 3, baseDelayMs: 1, onRetry }); + + expect(result).toBe("ok"); + expect(onRetry).toHaveBeenCalledWith(expect.any(Error), 1, 1); + }); + + it("rethrows once the attempts are exhausted", async () => { + const operation = vi.fn().mockRejectedValue(new Error("persistent")); + + await expect(retryWithBackoff(operation, { maxAttempts: 3, baseDelayMs: 1, onRetry: vi.fn() })).rejects.toThrow("persistent"); + expect(operation).toHaveBeenCalledTimes(3); + }); + + it("rethrows immediately when shouldRethrow matches", async () => { + const operation = vi.fn().mockRejectedValue(new Error("fatal")); + + await expect(retryWithBackoff(operation, { maxAttempts: 3, baseDelayMs: 1, shouldRethrow: () => true, onRetry: vi.fn() })).rejects.toThrow("fatal"); + expect(operation).toHaveBeenCalledTimes(1); + }); + + it("caps the backoff delay at maxDelayMs", async () => { + const operation = vi.fn().mockRejectedValueOnce(new Error("a")).mockRejectedValueOnce(new Error("b")).mockResolvedValue("ok"); + const onRetry = vi.fn(); + + await retryWithBackoff(operation, { maxAttempts: 5, baseDelayMs: 2, maxDelayMs: 3, onRetry }); + + expect(onRetry.mock.calls.map(call => call[2])).toEqual([2, 3]); + }); +}); diff --git a/apps/chain-indexer/src/lib/retry-with-backoff/retry-with-backoff.ts b/apps/chain-indexer/src/lib/retry-with-backoff/retry-with-backoff.ts new file mode 100644 index 0000000000..3c209d9432 --- /dev/null +++ b/apps/chain-indexer/src/lib/retry-with-backoff/retry-with-backoff.ts @@ -0,0 +1,35 @@ +export interface RetryWithBackoffOptions { + maxAttempts: number; + baseDelayMs: number; + maxDelayMs?: number; + /** Errors for which a retry cannot help (fatal conditions, shutdown in progress); they propagate immediately. */ + shouldRethrow?: (error: unknown) => boolean; + onRetry: (error: unknown, attempt: number, delayMs: number) => void; +} + +export async function retryWithBackoff(operation: () => Promise, options: RetryWithBackoffOptions): Promise { + let attempt = 0; + + while (true) { + attempt++; + try { + return await operation(); + } catch (error) { + if (options.shouldRethrow?.(error) || attempt >= options.maxAttempts) { + throw error; + } + + const uncappedDelayMs = options.baseDelayMs * 2 ** (attempt - 1); + const delayMs = options.maxDelayMs === undefined ? uncappedDelayMs : Math.min(uncappedDelayMs, options.maxDelayMs); + options.onRetry(error, attempt, delayMs); + await delay(delayMs); + } + } +} + +/** Global setTimeout rather than node:timers/promises so tests can advance the backoff with vitest fake timers, which do not intercept node:timers/promises. */ +function delay(ms: number): Promise { + return new Promise(resolve => { + setTimeout(resolve, ms); + }); +} diff --git a/apps/chain-indexer/src/pipeline/backfill-planner.spec.ts b/apps/chain-indexer/src/pipeline/backfill-planner.spec.ts new file mode 100644 index 0000000000..68670e4045 --- /dev/null +++ b/apps/chain-indexer/src/pipeline/backfill-planner.spec.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; + +import { planBackfill } from "@src/pipeline/backfill-planner"; + +describe(planBackfill.name, () => { + it("runs the full range when there is no checkpoint", () => { + const plan = planBackfill({ fromHeight: 100, toHeight: 200, checkpointHeight: null, tipHeight: 1_000 }); + + expect(plan).toEqual({ kind: "run", startHeight: 100, endHeight: 200 }); + }); + + it("resumes after the checkpoint when one exists mid-range", () => { + const plan = planBackfill({ fromHeight: 100, toHeight: 200, checkpointHeight: 150, tipHeight: 1_000 }); + + expect(plan).toEqual({ kind: "run", startHeight: 151, endHeight: 200 }); + }); + + it("reports already-complete when the checkpoint reached the end of the range", () => { + const plan = planBackfill({ fromHeight: 100, toHeight: 200, checkpointHeight: 200, tipHeight: 1_000 }); + + expect(plan).toEqual({ kind: "already-complete" }); + }); + + it("reports already-complete even when a lagging node returns a stale tip below the range end", () => { + const plan = planBackfill({ fromHeight: 100, toHeight: 200, checkpointHeight: 200, tipHeight: 150 }); + + expect(plan).toEqual({ kind: "already-complete" }); + }); + + it("rejects a range ending above the chain tip", () => { + const plan = planBackfill({ fromHeight: 100, toHeight: 2_000, checkpointHeight: null, tipHeight: 1_000 }); + + expect(plan).toEqual({ kind: "invalid", reason: "BACKFILL_TO_HEIGHT 2000 is above the chain tip 1000" }); + }); + + it("runs a single-block range", () => { + const plan = planBackfill({ fromHeight: 100, toHeight: 100, checkpointHeight: null, tipHeight: 1_000 }); + + expect(plan).toEqual({ kind: "run", startHeight: 100, endHeight: 100 }); + }); +}); diff --git a/apps/chain-indexer/src/pipeline/backfill-planner.ts b/apps/chain-indexer/src/pipeline/backfill-planner.ts new file mode 100644 index 0000000000..739265f1a0 --- /dev/null +++ b/apps/chain-indexer/src/pipeline/backfill-planner.ts @@ -0,0 +1,29 @@ +export interface BackfillPlanInput { + fromHeight: number; + toHeight: number; + checkpointHeight: number | null; + tipHeight: number; +} + +export type BackfillPlan = { kind: "run"; startHeight: number; endHeight: number } | { kind: "already-complete" } | { kind: "invalid"; reason: string }; + +/** + * A range above the chain tip is rejected rather than clamped: clamping would mark the range's + * checkpoint complete for heights that were never indexed. Completion is checked first so a + * re-run of a finished range stays a no-op even when a lagging RPC node reports a stale tip. + */ +export function planBackfill(input: BackfillPlanInput): BackfillPlan { + if (input.checkpointHeight !== null && input.checkpointHeight >= input.toHeight) { + return { kind: "already-complete" }; + } + + if (input.toHeight > input.tipHeight) { + return { kind: "invalid", reason: `BACKFILL_TO_HEIGHT ${input.toHeight} is above the chain tip ${input.tipHeight}` }; + } + + return { + kind: "run", + startHeight: input.checkpointHeight !== null ? input.checkpointHeight + 1 : input.fromHeight, + endHeight: input.toHeight + }; +} diff --git a/apps/chain-indexer/src/pipeline/backfill-runner.service.spec.ts b/apps/chain-indexer/src/pipeline/backfill-runner.service.spec.ts new file mode 100644 index 0000000000..9d81537c25 --- /dev/null +++ b/apps/chain-indexer/src/pipeline/backfill-runner.service.spec.ts @@ -0,0 +1,254 @@ +import { setTimeout as delay } from "node:timers/promises"; +import { describe, expect, it, vi } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import { envSchema } from "@src/config/env.config"; +import { Blocks, IndexerState } from "@src/db/schema"; +import { BackfillRunnerService } from "@src/pipeline/backfill-runner.service"; +import type { BlockCommitterService } from "@src/pipeline/block-committer.service"; +import type { BlockDecoderService } from "@src/pipeline/block-decoder.service"; +import type { DecodedBlock } from "@src/pipeline/decoded-block"; +import { RunnerInterruptedError } from "@src/pipeline/runner-interrupted-error"; +import type { ChainDatabase } from "@src/providers/db.provider"; +import type { LoggerService } from "@src/providers/logging.provider"; +import type { RpcClientPool } from "@src/rpc/rpc-client-pool.service"; +import type { RpcBlockResult } from "@src/rpc/rpc-types"; + +describe(BackfillRunnerService.name, () => { + it("commits blocks in ascending order even when fetches resolve out of order", async () => { + const { runner, committer } = setup({ + fromHeight: 1, + toHeight: 5, + batchSize: 5, + concurrency: 5, + fetchDelayMs: height => (6 - height) * 5 + }); + + await runner.start(); + + expect(committer.commitBatch).toHaveBeenCalledTimes(1); + expect(committedHeights(committer)).toEqual([[1, 2, 3, 4, 5]]); + }); + + it("never fetches more blocks in parallel than the configured concurrency", async () => { + const { runner, maxObservedConcurrency } = setup({ fromHeight: 1, toHeight: 10, batchSize: 10, concurrency: 3, fetchDelayMs: () => 2 }); + + await runner.start(); + + expect(maxObservedConcurrency()).toBeLessThanOrEqual(3); + }); + + it("commits in batches of the configured size under the range-scoped stream", async () => { + const { runner, committer } = setup({ fromHeight: 1, toHeight: 5, batchSize: 2 }); + + await runner.start(); + + expect(committedHeights(committer)).toEqual([[1, 2], [3, 4], [5]]); + expect(committer.commitBatch.mock.calls.map(call => call[1])).toEqual([{ stream: "backfill:1-5" }, { stream: "backfill:1-5" }, { stream: "backfill:1-5" }]); + }); + + it("resumes after the checkpoint and verifies continuity against the checkpoint block", async () => { + const { runner, committer, pool } = setup({ + fromHeight: 1, + toHeight: 5, + checkpointHeight: 3, + seedBlock: { height: 3, hash: heightHash(3) } + }); + + await runner.start(); + + expect(pool.getBlock).not.toHaveBeenCalledWith(3); + expect(committedHeights(committer)).toEqual([[4, 5]]); + }); + + it("throws when the checkpoint block is missing on resume", async () => { + const { runner, committer } = setup({ fromHeight: 1, toHeight: 5, checkpointHeight: 3 }); + + await expect(runner.start()).rejects.toThrow("Checkpoint block 3 is missing"); + expect(committer.commitBatch).not.toHaveBeenCalled(); + }); + + it("exits without fetching anything when the checkpoint already covers the range", async () => { + const { runner, committer, pool, logger } = setup({ fromHeight: 1, toHeight: 5, checkpointHeight: 5 }); + + await runner.start(); + + expect(pool.getBlock).not.toHaveBeenCalled(); + expect(committer.commitBatch).not.toHaveBeenCalled(); + expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ event: "BACKFILL_ALREADY_COMPLETE" })); + }); + + it("fails when the range ends above the chain tip", async () => { + const { runner, committer } = setup({ fromHeight: 1, toHeight: 5, tipHeight: 3 }); + + await expect(runner.start()).rejects.toThrow("BACKFILL_TO_HEIGHT 5 is above the chain tip 3"); + expect(committer.commitBatch).not.toHaveBeenCalled(); + }); + + it("halts without committing when the parent-hash chain breaks", async () => { + const { runner, committer } = setup({ fromHeight: 1, toHeight: 5, brokenParentAtHeight: 3 }); + + await expect(runner.start()).rejects.toThrow("Parent hash mismatch at height 3; halting backfill"); + expect(committer.commitBatch).not.toHaveBeenCalled(); + }); + + it("retries a failed fetch and still commits the block", async () => { + vi.useFakeTimers(); + + try { + const { runner, committer, logger } = setup({ fromHeight: 1, toHeight: 2, failFetchOnceAtHeight: 2 }); + + const started = runner.start(); + await vi.runAllTimersAsync(); + await started; + + expect(committedHeights(committer)).toEqual([[1, 2]]); + expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ event: "BACKFILL_FETCH_RETRY", height: 2, attempt: 1 })); + } finally { + vi.useRealTimers(); + } + }); + + it("logs a completion summary with throughput counters", async () => { + const { runner, logger } = setup({ fromHeight: 1, toHeight: 5, txCountPerBlock: 2 }); + + await runner.start(); + + expect(logger.info).toHaveBeenCalledWith( + expect.objectContaining({ + event: "BACKFILL_COMPLETED", + stream: "backfill:1-5", + startHeight: 1, + endHeight: 5, + blocksCommitted: 5, + transactionsCommitted: 10, + durationMs: expect.any(Number), + blocksPerSecond: expect.any(Number) + }) + ); + }); + + it("rejects with RunnerInterruptedError when stopped before the range completes", async () => { + const { runner, committer } = setup({ fromHeight: 1, toHeight: 10, batchSize: 2, concurrency: 2 }); + committer.commitBatch.mockImplementationOnce(async () => { + await runner.dispose(); + }); + + await expect(runner.start()).rejects.toThrow(RunnerInterruptedError); + expect(committedHeights(committer)).toEqual([[1, 2]]); + }); + + it("reports completion when stopped during the final commit that covers the range", async () => { + const { runner, committer, logger } = setup({ fromHeight: 1, toHeight: 5, batchSize: 5, concurrency: 5 }); + committer.commitBatch.mockImplementationOnce(async () => { + await runner.dispose(); + }); + + await expect(runner.start()).resolves.toBeUndefined(); + expect(committedHeights(committer)).toEqual([[1, 2, 3, 4, 5]]); + expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ event: "BACKFILL_COMPLETED" })); + }); + + function setup(input: { + fromHeight: number; + toHeight: number; + batchSize?: number; + concurrency?: number; + tipHeight?: number; + checkpointHeight?: number; + seedBlock?: { height: number; hash: Buffer }; + fetchDelayMs?: (height: number) => number; + failFetchOnceAtHeight?: number; + brokenParentAtHeight?: number; + txCountPerBlock?: number; + }) { + const config = envSchema.parse({ + POSTGRES_DB_URI: "postgres://unit:unit@localhost:5432/unit", + INDEXER_ROLE: "backfill", + BACKFILL_FROM_HEIGHT: String(input.fromHeight), + BACKFILL_TO_HEIGHT: String(input.toHeight), + BACKFILL_BATCH_SIZE: String(input.batchSize ?? 200), + BACKFILL_CONCURRENCY: String(input.concurrency ?? 10) + }); + + const dbFake = { + select: () => ({ + from: (table: unknown) => ({ + where: () => { + if (table === IndexerState && input.checkpointHeight !== undefined) { + return Promise.resolve([{ stream: `backfill:${input.fromHeight}-${input.toHeight}`, lastHeight: input.checkpointHeight }]); + } + if (table === Blocks && input.seedBlock) { + return Promise.resolve([input.seedBlock]); + } + return Promise.resolve([]); + } + }) + }) + }; + + let activeFetches = 0; + let maxActiveFetches = 0; + let failedOnce = false; + const pool = mock(); + pool.getTipHeight.mockResolvedValue(input.tipHeight ?? 1_000); + pool.getBlock.mockImplementation(async height => { + if (input.failFetchOnceAtHeight === height && !failedOnce) { + failedOnce = true; + throw new AggregateError([new Error("all nodes failed")], `Failed to fetch block ${height}`); + } + + activeFetches++; + maxActiveFetches = Math.max(maxActiveFetches, activeFetches); + await delay(input.fetchDelayMs?.(height) ?? 0); + activeFetches--; + return { block: { header: { height: String(height) } } } as RpcBlockResult; + }); + pool.getBlockResults.mockResolvedValue({ height: "0", txs_results: null }); + + const decoder = mock(); + decoder.decode.mockImplementation(block => { + const height = parseInt(block.block.header.height); + return buildDecodedBlock(height, { + parentHash: input.brokenParentAtHeight === height ? Buffer.from("bogus") : heightHash(height - 1), + txCount: input.txCountPerBlock ?? 0 + }); + }); + + const committer = mock(); + committer.commitBatch.mockResolvedValue(undefined); + + const logger = mock(); + + const runner = new BackfillRunnerService(dbFake as unknown as ChainDatabase, pool, decoder, committer, config, logger); + + return { runner, committer, pool, logger, maxObservedConcurrency: () => maxActiveFetches }; + } + + function committedHeights(committer: { commitBatch: { mock: { calls: unknown[][] } } }) { + return committer.commitBatch.mock.calls.map(call => (call[0] as DecodedBlock[]).map(block => block.height)); + } + + function buildDecodedBlock(height: number, options: { parentHash: Buffer; txCount: number }): DecodedBlock { + return { + height, + datetime: new Date("2026-08-11T00:00:00Z"), + hash: heightHash(height), + parentHash: options.parentHash, + proposerAddress: "PROPOSER", + transactions: Array.from({ length: options.txCount }, (_, index) => ({ + index, + hash: Buffer.from(`tx-${height}-${index}`), + code: 0, + gasUsed: 0, + gasWanted: 0, + fee: [], + messages: [] + })) + }; + } + + function heightHash(height: number): Buffer { + return Buffer.from(`hash-${height}`); + } +}); diff --git a/apps/chain-indexer/src/pipeline/backfill-runner.service.ts b/apps/chain-indexer/src/pipeline/backfill-runner.service.ts new file mode 100644 index 0000000000..1961726277 --- /dev/null +++ b/apps/chain-indexer/src/pipeline/backfill-runner.service.ts @@ -0,0 +1,241 @@ +import { eq } from "drizzle-orm"; +import { inject, singleton } from "tsyringe"; + +import type { EnvConfig } from "@src/config/env.config"; +import { Blocks, IndexerState } from "@src/db/schema"; +import { retryWithBackoff } from "@src/lib/retry-with-backoff/retry-with-backoff"; +import { planBackfill } from "@src/pipeline/backfill-planner"; +import { BlockCommitterService } from "@src/pipeline/block-committer.service"; +import { BlockDecoderService } from "@src/pipeline/block-decoder.service"; +import { ChainContinuityError } from "@src/pipeline/chain-continuity-error"; +import type { DecodedBlock } from "@src/pipeline/decoded-block"; +import { RunnerInterruptedError } from "@src/pipeline/runner-interrupted-error"; +import { retryTransient } from "@src/pipeline/transient-retry"; +import { APP_CONFIG } from "@src/providers/app-config.provider"; +import type { ChainDatabase } from "@src/providers/db.provider"; +import { CHAIN_DB } from "@src/providers/db.provider"; +import { LoggerService } from "@src/providers/logging.provider"; +import { RpcClientPool } from "@src/rpc/rpc-client-pool.service"; + +const FETCH_RETRY_MAX_ATTEMPTS = 5; +const FETCH_RETRY_BASE_MS = 1_000; + +@singleton() +export class BackfillRunnerService { + readonly #db: ChainDatabase; + readonly #pool: RpcClientPool; + readonly #decoder: BlockDecoderService; + readonly #committer: BlockCommitterService; + readonly #config: EnvConfig; + readonly #logger: LoggerService; + + #stopped = false; + #lastHash: Buffer | null = null; + + constructor( + @inject(CHAIN_DB) db: ChainDatabase, + @inject(RpcClientPool) pool: RpcClientPool, + @inject(BlockDecoderService) decoder: BlockDecoderService, + @inject(BlockCommitterService) committer: BlockCommitterService, + @inject(APP_CONFIG) config: EnvConfig, + @inject(LoggerService) logger: LoggerService + ) { + this.#db = db; + this.#pool = pool; + this.#decoder = decoder; + this.#committer = committer; + this.#config = config; + this.#logger = logger; + this.#logger.setContext("BACKFILL"); + } + + async start(): Promise { + let completed: boolean; + + try { + completed = await this.#run(); + } catch (error) { + if (this.#stopped) { + throw new RunnerInterruptedError("Backfill stopped before completing the range", { cause: error }); + } + throw error; + } + + if (!completed) { + throw new RunnerInterruptedError("Backfill stopped before completing the range"); + } + } + + async dispose(): Promise { + this.#stopped = true; + } + + async #run(): Promise { + const { BACKFILL_FROM_HEIGHT: fromHeight, BACKFILL_TO_HEIGHT: toHeight } = this.#config; + + if (fromHeight === undefined || toHeight === undefined) { + throw new Error("BACKFILL_FROM_HEIGHT and BACKFILL_TO_HEIGHT are required for the backfill role"); + } + + const stream = `backfill:${fromHeight}-${toHeight}`; + const [checkpointHeight, tipHeight] = await Promise.all([ + this.#retryTransient(() => this.#getCheckpointHeight(stream), { event: "BACKFILL_CHECKPOINT_READ_RETRY" }), + this.#retryTransient(() => this.#pool.getTipHeight(), { event: "BACKFILL_TIP_FETCH_RETRY" }) + ]); + const plan = planBackfill({ fromHeight, toHeight, checkpointHeight, tipHeight }); + + if (plan.kind === "invalid") { + this.#logger.error({ event: "BACKFILL_INVALID_RANGE", reason: plan.reason }); + throw new Error(plan.reason); + } + + if (plan.kind === "already-complete") { + this.#logger.info({ event: "BACKFILL_ALREADY_COMPLETE", stream, checkpointHeight }); + return true; + } + + await this.#seedContinuityHash(plan.startHeight, checkpointHeight !== null); + this.#logger.info({ event: "BACKFILL_STARTED", network: this.#config.NETWORK, stream, startHeight: plan.startHeight, endHeight: plan.endHeight }); + + return await this.#backfillRange(plan.startHeight, plan.endHeight, stream); + } + + /** + * Fetches up to BACKFILL_CONCURRENCY blocks in parallel while consuming heights strictly in + * order, so batches handed to the committer are contiguous and ordered by construction. + * Prefetched promises get a no-op catch at insertion: a rejection settling before the loop + * reaches its height would otherwise crash the process as an unhandled rejection; the real + * rejection still surfaces when the loop awaits that height. + * + * Returns whether the whole range committed. Completion is tracked by the last committed height + * rather than the stopped flag, so a shutdown landing during the final commit still reports the + * range as done instead of failing the Job for a spurious retry. + */ + async #backfillRange(startHeight: number, endHeight: number, stream: string): Promise { + const startedAt = Date.now(); + const inflight = new Map>(); + let fetchHead = startHeight; + let blocksCommitted = 0; + let transactionsCommitted = 0; + let lastCommittedHeight = startHeight - 1; + let batch: DecodedBlock[] = []; + + const fillFetchWindow = () => { + while (fetchHead <= endHeight && inflight.size < this.#config.BACKFILL_CONCURRENCY) { + const height = fetchHead; + const prefetched = this.#fetchAndDecode(height); + prefetched.catch(() => undefined); + inflight.set(height, prefetched); + fetchHead++; + } + }; + + try { + for (let height = startHeight; height <= endHeight && !this.#stopped; height++) { + fillFetchWindow(); + const decoded = await inflight.get(height)!; + inflight.delete(height); + + this.#verifyContinuity(decoded); + this.#lastHash = decoded.hash; + batch.push(decoded); + fillFetchWindow(); + + if (batch.length >= this.#config.BACKFILL_BATCH_SIZE || height === endHeight) { + const currentBatch = batch; + await this.#retryTransient(() => this.#committer.commitBatch(currentBatch, { stream }), { event: "BACKFILL_COMMIT_RETRY", height }); + blocksCommitted += batch.length; + transactionsCommitted += batch.reduce((sum, block) => sum + block.transactions.length, 0); + lastCommittedHeight = height; + batch = []; + this.#logger.info({ event: "BACKFILL_PROGRESS", height, endHeight, blocksCommitted }); + } + } + } finally { + await Promise.allSettled([...inflight.values()]); + } + + if (lastCommittedHeight < endHeight) { + return false; + } + + const durationMs = Date.now() - startedAt; + this.#logger.info({ + event: "BACKFILL_COMPLETED", + stream, + startHeight, + endHeight, + blocksCommitted, + transactionsCommitted, + durationMs, + blocksPerSecond: durationMs > 0 ? Math.round((blocksCommitted / durationMs) * 1_000 * 100) / 100 : blocksCommitted + }); + + return true; + } + + /** Retriable steps (checkpoint reads, tip fetches, idempotent batch commits) survive transient blips instead of failing the whole multi-hour Job; fatal errors propagate. */ + async #retryTransient(operation: () => Promise, logContext: { event: string; height?: number }): Promise { + return await retryTransient(operation, { isStopped: () => this.#stopped, logger: this.#logger, logContext }); + } + + /** A pool AggregateError means every RPC endpoint already failed once, so retries back off before another full sweep. */ + async #fetchAndDecode(height: number): Promise { + return await retryWithBackoff( + async () => { + const [block, blockResults] = await Promise.all([this.#pool.getBlock(height), this.#pool.getBlockResults(height)]); + return this.#decoder.decode(block, blockResults); + }, + { + maxAttempts: FETCH_RETRY_MAX_ATTEMPTS, + baseDelayMs: FETCH_RETRY_BASE_MS, + shouldRethrow: () => this.#stopped, + onRetry: (error, attempt, delayMs) => this.#logger.warn({ event: "BACKFILL_FETCH_RETRY", height, attempt, delayMs, error }) + } + ); + } + + #verifyContinuity(block: DecodedBlock): void { + if (this.#lastHash && block.parentHash && !block.parentHash.equals(this.#lastHash)) { + this.#logger.error({ + event: "BACKFILL_CONTINUITY_BROKEN", + height: block.height, + expectedParentHash: this.#lastHash.toString("hex"), + actualParentHash: block.parentHash.toString("hex") + }); + throw new ChainContinuityError(`Parent hash mismatch at height ${block.height}; halting backfill`); + } + } + + /** + * The parent-hash chain is seeded from the block before the start height. On resume that block + * was committed by this stream's checkpoint and must exist; on a fresh start it may have been + * committed by sync or another backfill, and its absence just leaves the first block unverified. + */ + async #seedContinuityHash(startHeight: number, isResume: boolean): Promise { + const [previousBlock] = await this.#retryTransient( + () => + this.#db + .select() + .from(Blocks) + .where(eq(Blocks.height, startHeight - 1)), + { event: "BACKFILL_SEED_READ_RETRY", height: startHeight - 1 } + ); + + if (previousBlock) { + this.#lastHash = previousBlock.hash; + return; + } + + if (isResume) { + throw new Error(`Checkpoint block ${startHeight - 1} is missing; cannot verify continuity on resume`); + } + + this.#lastHash = null; + } + + async #getCheckpointHeight(stream: string): Promise { + const [state] = await this.#db.select().from(IndexerState).where(eq(IndexerState.stream, stream)); + return state?.lastHeight ?? null; + } +} diff --git a/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts b/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts index 543ca4ab4d..3da89ecba0 100644 --- a/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts +++ b/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts @@ -1,6 +1,8 @@ +import type { SQL } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; import { describe, expect, it } from "vitest"; -import { Messages, MessageTypes } from "@src/db/schema"; +import { Blocks, IndexerState, Messages, MessageTypes } from "@src/db/schema"; import { BlockCommitterService } from "@src/pipeline/block-committer.service"; import type { DecodedBlock } from "@src/pipeline/decoded-block"; import type { ChainDatabase } from "@src/providers/db.provider"; @@ -41,9 +43,66 @@ describe(BlockCommitterService.name, () => { expect(insertedRows.find(call => call.table === Messages)?.rows).toEqual([expect.objectContaining({ typeId: 9 })]); }); + it("advances the sync stream checkpoint when committing a single block", async () => { + const { committer, insertedRows } = setup({ selectResults: [[{ id: 7, type: MSG_SEND }]] }); + + await committer.commit(buildBlock([MSG_SEND])); + + expect(insertedRows.find(call => call.table === IndexerState)?.rows).toEqual(expect.objectContaining({ stream: "sync", lastHeight: 10 })); + }); + + describe("commitBatch", () => { + it("commits all blocks and advances the given stream checkpoint to the batch's last height", async () => { + const { committer, insertedRows } = setup({ selectResults: [[{ id: 7, type: MSG_SEND }]] }); + + await committer.commitBatch([buildBlock([MSG_SEND], 10), buildBlock([MSG_SEND], 11), buildBlock([MSG_SEND], 12)], { stream: "backfill:10-12" }); + + expect(insertedRows.find(call => call.table === Blocks)?.rows).toHaveLength(3); + expect(insertedRows.find(call => call.table === IndexerState)?.rows).toEqual(expect.objectContaining({ stream: "backfill:10-12", lastHeight: 12 })); + }); + + it("throws on a non-contiguous batch before writing anything", async () => { + const { committer, insertedRows } = setup(); + + await expect(committer.commitBatch([buildBlock([MSG_SEND], 10), buildBlock([MSG_SEND], 12)], { stream: "backfill:10-12" })).rejects.toThrow( + "Non-contiguous batch: expected height 11 at position 1, got 12" + ); + expect(insertedRows).toEqual([]); + }); + + it("does nothing for an empty batch", async () => { + const { committer, insertedRows } = setup(); + + await committer.commitBatch([], { stream: "backfill:10-12" }); + + expect(insertedRows).toEqual([]); + }); + + it("only ever moves the checkpoint forward on conflict, so concurrent writers cannot regress it", async () => { + const { committer, conflictUpdates } = setup({ selectResults: [[{ id: 7, type: MSG_SEND }]] }); + + await committer.commitBatch([buildBlock([MSG_SEND], 10)], { stream: "backfill:10-10" }); + + const checkpointSet = conflictUpdates.find(call => call.table === IndexerState)?.config.set as { lastHeight: SQL }; + expect(new PgDialect().sqlToQuery(checkpointSet.lastHeight).sql).toBe('GREATEST("indexer_state"."last_height", EXCLUDED.last_height)'); + }); + + it("splits large row sets into multiple inserts within the same transaction", async () => { + const { committer, insertedRows } = setup({ selectResults: [[{ id: 7, type: MSG_SEND }]] }); + const manyMessages = Array.from({ length: 2_001 }, () => MSG_SEND); + + await committer.commitBatch([buildBlock(manyMessages, 10)], { stream: "backfill:10-10" }); + + const messageInserts = insertedRows.filter(call => call.table === Messages); + expect(messageInserts).toHaveLength(2); + expect(messageInserts.map(call => (call.rows as unknown[]).length)).toEqual([2_000, 1]); + }); + }); + function setup(input?: { selectResults?: Array>; insertReturning?: Array<{ id: number; type: string }> }) { const selectResults = [...(input?.selectResults ?? [[]])]; const insertedRows: Array<{ table: unknown; rows: unknown }> = []; + const conflictUpdates: Array<{ table: unknown; config: { set: unknown } }> = []; const dbFake = { select: () => ({ from: () => ({ where: () => Promise.resolve(selectResults.shift() ?? []) }) }), @@ -55,7 +114,10 @@ describe(BlockCommitterService.name, () => { Object.assign(Promise.resolve(), { returning: () => Promise.resolve(input?.insertReturning ?? []) }), - onConflictDoUpdate: () => Promise.resolve() + onConflictDoUpdate: (config: { set: unknown }) => { + conflictUpdates.push({ table, config }); + return Promise.resolve(); + } }; } }), @@ -63,12 +125,12 @@ describe(BlockCommitterService.name, () => { }; const committer = new BlockCommitterService(dbFake as unknown as ChainDatabase); - return { committer, insertedRows }; + return { committer, insertedRows, conflictUpdates }; } - function buildBlock(typeUrls: string[]): DecodedBlock { + function buildBlock(typeUrls: string[], height = 10): DecodedBlock { return { - height: 10, + height, datetime: new Date("2026-08-11T00:00:00Z"), hash: Buffer.from("aa".repeat(32), "hex"), parentHash: null, diff --git a/apps/chain-indexer/src/pipeline/block-committer.service.ts b/apps/chain-indexer/src/pipeline/block-committer.service.ts index 056c467cf1..72c01f8f42 100644 --- a/apps/chain-indexer/src/pipeline/block-committer.service.ts +++ b/apps/chain-indexer/src/pipeline/block-committer.service.ts @@ -1,4 +1,5 @@ -import { inArray } from "drizzle-orm"; +import { inArray, sql } from "drizzle-orm"; +import chunk from "lodash/chunk"; import { inject, singleton } from "tsyringe"; import { Blocks, IndexerState, Messages, MessageTypes, Transactions } from "@src/db/schema"; @@ -8,6 +9,9 @@ import { CHAIN_DB } from "@src/providers/db.provider"; export const SYNC_STREAM = "sync"; +/** Keeps multi-row inserts well under postgres.js's ~65k bind-parameter limit when batches span hundreds of blocks. */ +const INSERT_CHUNK_SIZE = 2_000; + @singleton() export class BlockCommitterService { readonly #db: ChainDatabase; @@ -18,61 +22,96 @@ export class BlockCommitterService { } async commit(block: DecodedBlock): Promise { - const typeIds = await this.#internMessageTypes(block); + await this.commitBatch([block], { stream: SYNC_STREAM }); + } + + /** + * Commits contiguous blocks and the checkpoint advance in one transaction, so the checkpoint + * never points past uncommitted data. Inserts are conflict-ignoring and the checkpoint only + * moves forward, so concurrent writers on the same stream (e.g. two pods overlapping during a + * rolling deploy) duplicate work but cannot corrupt data or regress the checkpoint. + */ + async commitBatch(blocks: DecodedBlock[], options: { stream: string }): Promise { + if (blocks.length === 0) { + return; + } - const transactionRows = block.transactions.map(tx => ({ + this.#verifyContiguous(blocks); + const typeIds = await this.#internMessageTypes(blocks); + + const blockRows = blocks.map(block => ({ height: block.height, - index: tx.index, - hash: tx.hash, - code: tx.code, - gasUsed: tx.gasUsed, - gasWanted: tx.gasWanted, - fee: tx.fee + datetime: block.datetime, + hash: block.hash, + parentHash: block.parentHash, + proposerAddress: block.proposerAddress, + txCount: block.transactions.length })); - const messageRows = block.transactions.flatMap(tx => - tx.messages.map(message => ({ + const transactionRows = blocks.flatMap(block => + block.transactions.map(tx => ({ height: block.height, - txIndex: tx.index, - index: message.index, - typeId: typeIds.get(message.typeUrl) as number, - body: message.body + index: tx.index, + hash: tx.hash, + code: tx.code, + gasUsed: tx.gasUsed, + gasWanted: tx.gasWanted, + fee: tx.fee })) ); - await this.#db.transaction(async tx => { - await tx - .insert(Blocks) - .values({ + const messageRows = blocks.flatMap(block => + block.transactions.flatMap(tx => + tx.messages.map(message => ({ height: block.height, - datetime: block.datetime, - hash: block.hash, - parentHash: block.parentHash, - proposerAddress: block.proposerAddress, - txCount: block.transactions.length - }) - .onConflictDoNothing(); - - if (transactionRows.length > 0) { - await tx.insert(Transactions).values(transactionRows).onConflictDoNothing(); + txIndex: tx.index, + index: message.index, + typeId: typeIds.get(message.typeUrl) as number, + body: message.body + })) + ) + ); + + const lastHeight = blocks[blocks.length - 1].height; + + await this.#db.transaction(async tx => { + for (const blockChunk of chunk(blockRows, INSERT_CHUNK_SIZE)) { + await tx.insert(Blocks).values(blockChunk).onConflictDoNothing(); } - if (messageRows.length > 0) { - await tx.insert(Messages).values(messageRows).onConflictDoNothing(); + for (const transactionChunk of chunk(transactionRows, INSERT_CHUNK_SIZE)) { + await tx.insert(Transactions).values(transactionChunk).onConflictDoNothing(); + } + + for (const messageChunk of chunk(messageRows, INSERT_CHUNK_SIZE)) { + await tx.insert(Messages).values(messageChunk).onConflictDoNothing(); } await tx .insert(IndexerState) - .values({ stream: SYNC_STREAM, lastHeight: block.height, updatedAt: new Date() }) + .values({ stream: options.stream, lastHeight, updatedAt: new Date() }) .onConflictDoUpdate({ target: IndexerState.stream, - set: { lastHeight: block.height, updatedAt: new Date() } + set: { lastHeight: sql`GREATEST(${IndexerState.lastHeight}, EXCLUDED.last_height)`, updatedAt: new Date() } }); }); } - async #internMessageTypes(block: DecodedBlock): Promise> { - const typeUrls = new Set(block.transactions.flatMap(tx => tx.messages.map(message => message.typeUrl))); + /** The checkpoint advances to the batch's last height, which is only correct when the batch has no gaps or reordering. */ + #verifyContiguous(blocks: DecodedBlock[]): void { + const baseHeight = blocks[0].height; + + blocks.forEach((block, index) => { + const expectedHeight = baseHeight + index; + + if (block.height !== expectedHeight) { + throw new Error(`Non-contiguous batch: expected height ${expectedHeight} at position ${index}, got ${block.height}`); + } + }); + } + + async #internMessageTypes(blocks: DecodedBlock[]): Promise> { + const typeUrls = new Set(blocks.flatMap(block => block.transactions.flatMap(tx => tx.messages.map(message => message.typeUrl)))); const uncached = [...typeUrls].filter(typeUrl => !this.#typeIds.has(typeUrl)); if (uncached.length > 0) { diff --git a/apps/chain-indexer/src/pipeline/chain-continuity-error.ts b/apps/chain-indexer/src/pipeline/chain-continuity-error.ts new file mode 100644 index 0000000000..8b2efb703b --- /dev/null +++ b/apps/chain-indexer/src/pipeline/chain-continuity-error.ts @@ -0,0 +1,2 @@ +/** Parent-hash continuity break; fatal by design so the process halts instead of committing a forked history. */ +export class ChainContinuityError extends Error {} diff --git a/apps/chain-indexer/src/pipeline/runner-interrupted-error.ts b/apps/chain-indexer/src/pipeline/runner-interrupted-error.ts new file mode 100644 index 0000000000..b14fb0cd16 --- /dev/null +++ b/apps/chain-indexer/src/pipeline/runner-interrupted-error.ts @@ -0,0 +1,6 @@ +/** + * A runner was asked to stop (SIGTERM / container disposal) before finishing its work. The process + * must exit non-zero so a K8s Job resumes from its checkpoint on the next attempt instead of being + * marked Complete with the range still unfinished. + */ +export class RunnerInterruptedError extends Error {} diff --git a/apps/chain-indexer/src/pipeline/sync-runner.service.ts b/apps/chain-indexer/src/pipeline/sync-runner.service.ts index 16350f8113..294eea08a5 100644 --- a/apps/chain-indexer/src/pipeline/sync-runner.service.ts +++ b/apps/chain-indexer/src/pipeline/sync-runner.service.ts @@ -1,38 +1,24 @@ import { eq } from "drizzle-orm"; import { setTimeout as delay } from "node:timers/promises"; -import type postgres from "postgres"; import { inject, singleton } from "tsyringe"; import type { EnvConfig } from "@src/config/env.config"; -import { PgClientService } from "@src/db/pg-client.service"; import { Blocks, IndexerState } from "@src/db/schema"; import { BlockCommitterService, SYNC_STREAM } from "@src/pipeline/block-committer.service"; import { BlockDecoderService } from "@src/pipeline/block-decoder.service"; +import { ChainContinuityError } from "@src/pipeline/chain-continuity-error"; import type { DecodedBlock } from "@src/pipeline/decoded-block"; +import { retryTransient } from "@src/pipeline/transient-retry"; import { APP_CONFIG } from "@src/providers/app-config.provider"; import type { ChainDatabase } from "@src/providers/db.provider"; import { CHAIN_DB } from "@src/providers/db.provider"; import { LoggerService } from "@src/providers/logging.provider"; import { RpcClientPool } from "@src/rpc/rpc-client-pool.service"; -/** Arbitrary but fixed application-wide key for the sync leader pg advisory lock. */ -const SYNC_LEADER_LOCK_KEY = 7_431_001; -const LEADERSHIP_RETRY_MS = 5_000; const PROGRESS_LOG_EVERY_BLOCKS = 100; -const LEADERSHIP_CHECK_EVERY_BLOCKS = 100; -const TRANSIENT_RETRY_MAX_ATTEMPTS = 10; -const TRANSIENT_RETRY_BASE_MS = 1_000; -const TRANSIENT_RETRY_MAX_MS = 30_000; - -/** Parent-hash continuity break; fatal by design so the process halts instead of committing a forked history. */ -export class ChainContinuityError extends Error {} - -/** The advisory-lock session was replaced (e.g. a transparent driver reconnect), so another process may hold leadership. */ -export class SyncLeadershipLostError extends Error {} @singleton() export class SyncRunnerService { - readonly #pgClient: PgClientService; readonly #db: ChainDatabase; readonly #pool: RpcClientPool; readonly #decoder: BlockDecoderService; @@ -41,12 +27,9 @@ export class SyncRunnerService { readonly #logger: LoggerService; #stopped = false; - #reserved: postgres.ReservedSql | undefined; - #leaderBackendPid: number | undefined; #lastHash: Buffer | null = null; constructor( - @inject(PgClientService) pgClient: PgClientService, @inject(CHAIN_DB) db: ChainDatabase, @inject(RpcClientPool) pool: RpcClientPool, @inject(BlockDecoderService) decoder: BlockDecoderService, @@ -54,7 +37,6 @@ export class SyncRunnerService { @inject(APP_CONFIG) config: EnvConfig, @inject(LoggerService) logger: LoggerService ) { - this.#pgClient = pgClient; this.#db = db; this.#pool = pool; this.#decoder = decoder; @@ -78,26 +60,14 @@ export class SyncRunnerService { async dispose(): Promise { this.#stopped = true; - this.#releaseReservedLockConnection(); - } - - /** release() may throw when the pg pool was ended first; the connection is already gone then, which is the goal. */ - #releaseReservedLockConnection(): void { - try { - this.#reserved?.release(); - } catch (error) { - this.#logger.debug({ event: "SYNC_LOCK_RELEASE_SKIPPED", error }); - } } async #run(): Promise { - await this.#acquireLeadership(); let nextHeight = await this.#resolveStartHeight(); this.#logger.info({ event: "SYNC_STARTED", network: this.#config.NETWORK, nextHeight }); while (!this.#stopped) { - await this.#assertLeadership(); - const tipHeight = await this.#retryTransient(() => this.#getTipHeight(), { event: "SYNC_TIP_FETCH_RETRY" }); + const tipHeight = await this.#retryTransient(() => this.#pool.getTipHeight(), { event: "SYNC_TIP_FETCH_RETRY" }); if (nextHeight > tipHeight) { await delay(this.#config.SYNC_POLL_INTERVAL_MS); @@ -108,34 +78,12 @@ export class SyncRunnerService { const height = nextHeight; await this.#retryTransient(() => this.#syncBlock(height), { event: "SYNC_BLOCK_RETRY", height }); nextHeight++; - - if (nextHeight % LEADERSHIP_CHECK_EVERY_BLOCKS === 0) { - await this.#assertLeadership(); - } } } } - /** Transient failures (RPC timeouts, connection drops) are retried with capped exponential backoff; persistent ones and fatal errors propagate and halt the process. */ async #retryTransient(operation: () => Promise, logContext: { event: string; height?: number }): Promise { - let attempt = 0; - - while (true) { - attempt++; - try { - return await operation(); - } catch (error) { - const isFatal = error instanceof ChainContinuityError || error instanceof SyncLeadershipLostError; - - if (this.#stopped || isFatal || attempt >= TRANSIENT_RETRY_MAX_ATTEMPTS) { - throw error; - } - - const delayMs = Math.min(TRANSIENT_RETRY_BASE_MS * 2 ** (attempt - 1), TRANSIENT_RETRY_MAX_MS); - this.#logger.warn({ ...logContext, attempt, delayMs, error }); - await delay(delayMs); - } - } + return await retryTransient(operation, { isStopped: () => this.#stopped, logger: this.#logger, logContext }); } async #syncBlock(height: number): Promise { @@ -165,38 +113,6 @@ export class SyncRunnerService { } } - async #acquireLeadership(): Promise { - this.#reserved = await this.#pgClient.client.reserve(); - - while (!this.#stopped) { - const [{ acquired }] = await this.#reserved`SELECT pg_try_advisory_lock(${SYNC_LEADER_LOCK_KEY}) AS acquired`; - - if (acquired) { - const [{ pid }] = await this.#reserved`SELECT pg_backend_pid() AS pid`; - this.#leaderBackendPid = pid; - this.#logger.info({ event: "SYNC_LEADERSHIP_ACQUIRED", backendPid: pid }); - return; - } - - this.#logger.info({ event: "SYNC_LEADERSHIP_WAITING" }); - await delay(LEADERSHIP_RETRY_MS); - } - } - - /** Advisory locks are session-scoped: a transparent driver reconnect creates a fresh session WITHOUT the lock, so the backend pid is re-checked to detect silent leadership loss before more blocks are committed. */ - async #assertLeadership(): Promise { - if (!this.#reserved) { - throw new SyncLeadershipLostError("Reserved advisory-lock connection is gone; halting sync"); - } - - const [{ pid }] = await this.#reserved`SELECT pg_backend_pid() AS pid`; - - if (pid !== this.#leaderBackendPid) { - this.#logger.error({ event: "SYNC_LEADERSHIP_LOST", expectedBackendPid: this.#leaderBackendPid, actualBackendPid: pid }); - throw new SyncLeadershipLostError(`Advisory-lock session changed (backend pid ${this.#leaderBackendPid} -> ${pid}); halting sync`); - } - } - async #resolveStartHeight(): Promise { const [state] = await this.#db.select().from(IndexerState).where(eq(IndexerState.stream, SYNC_STREAM)); @@ -210,11 +126,6 @@ export class SyncRunnerService { return this.#config.SYNC_START_HEIGHT; } - return await this.#getTipHeight(); - } - - async #getTipHeight(): Promise { - const status = await this.#pool.getStatus(); - return parseInt(status.sync_info.latest_block_height); + return await this.#pool.getTipHeight(); } } diff --git a/apps/chain-indexer/src/pipeline/transient-retry.ts b/apps/chain-indexer/src/pipeline/transient-retry.ts new file mode 100644 index 0000000000..16776f1d0f --- /dev/null +++ b/apps/chain-indexer/src/pipeline/transient-retry.ts @@ -0,0 +1,21 @@ +import { retryWithBackoff } from "@src/lib/retry-with-backoff/retry-with-backoff"; +import { ChainContinuityError } from "@src/pipeline/chain-continuity-error"; +import type { LoggerService } from "@src/providers/logging.provider"; + +const MAX_ATTEMPTS = 10; +const BASE_DELAY_MS = 1_000; +const MAX_DELAY_MS = 30_000; + +/** Transient failures (RPC timeouts, connection drops) are retried with capped exponential backoff; fatal pipeline errors and an in-progress shutdown propagate immediately. */ +export function retryTransient( + operation: () => Promise, + options: { isStopped: () => boolean; logger: LoggerService; logContext: { event: string; height?: number } } +): Promise { + return retryWithBackoff(operation, { + maxAttempts: MAX_ATTEMPTS, + baseDelayMs: BASE_DELAY_MS, + maxDelayMs: MAX_DELAY_MS, + shouldRethrow: error => options.isStopped() || error instanceof ChainContinuityError, + onRetry: (error, attempt, delayMs) => options.logger.warn({ ...options.logContext, attempt, delayMs, error }) + }); +} diff --git a/apps/chain-indexer/src/rpc/rpc-client-pool.service.spec.ts b/apps/chain-indexer/src/rpc/rpc-client-pool.service.spec.ts index fe43228414..e506dd097d 100644 --- a/apps/chain-indexer/src/rpc/rpc-client-pool.service.spec.ts +++ b/apps/chain-indexer/src/rpc/rpc-client-pool.service.spec.ts @@ -71,6 +71,13 @@ describe(RpcClientPool.name, () => { expect(fetchMock).toHaveBeenCalledTimes(2); }); + it("parses the tip height from the status payload", async () => { + const { pool, fetchMock } = setup(); + fetchMock.mockResolvedValue(jsonResponse({ result: { sync_info: { latest_block_height: "1234" } } })); + + await expect(pool.getTipHeight()).resolves.toBe(1234); + }); + it("requests block and block results with the height as a query parameter", async () => { const { pool, fetchMock } = setup(); fetchMock.mockResolvedValue(jsonResponse({ result: { height: "7" } })); diff --git a/apps/chain-indexer/src/rpc/rpc-client-pool.service.ts b/apps/chain-indexer/src/rpc/rpc-client-pool.service.ts index 60587e5b35..27f76063e4 100644 --- a/apps/chain-indexer/src/rpc/rpc-client-pool.service.ts +++ b/apps/chain-indexer/src/rpc/rpc-client-pool.service.ts @@ -46,6 +46,11 @@ export class RpcClientPool { return await this.#get("/status"); } + async getTipHeight(): Promise { + const status = await this.getStatus(); + return parseInt(status.sync_info.latest_block_height); + } + async getBlock(height: number): Promise { return await this.#get(`/block?height=${height}`); } From f13778e723e9fc1d6171deaa6b721563cc8afe76 Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Wed, 12 Aug 2026 19:56:47 +0530 Subject: [PATCH 04/14] feat(indexer): raw block archive in object storage (#3585) * feat(indexer): add raw block archive layout and zstd ndjson codec * feat(indexer): add gcs block archive service behind optional ARCHIVE_BUCKET * feat(indexer): archive staged raw blocks during live sync before commit * feat(indexer): source backfill blocks from the archive and compact ranges * test(indexer): cover the chain-id failure path in staged deletes * feat(indexer): allow overriding the gcs api endpoint for local emulators * chore(indexer): persist the chain-indexer local verify recipe * refactor(indexer): tighten archive invariants after code review * fix(indexer): stop malformed /status from poisoning the chain-id cache resolveChainId cached getStatus().then(onFulfilled, onRejected); a throw inside onFulfilled (a /status body missing node_info) rejected the cached promise without running the reset, so every later archive call failed with the stale rejection forever. Switch to .then().catch() so both a rejected fetch and a malformed payload clear the cache and let the retry re-fetch. Also share a fetchRawBlock helper between sync and backfill, free each range's buffer as soon as its chunk flush succeeds, and document the staged-single orphan window left by a crash between chunk put and delete. * docs(indexer): scope chain-indexer to chain data only State explicitly that the new chain-indexer owns chain-derived data only, and that off-chain provider data (status pinging, inventory, uptime, IP geolocation, GPU breakdown) belongs to apps/provider-inventory. Pricing and Keybase stay as the deliberate off-chain exception since the daily USD aggregates and validator records need them. Note the pending ownership move on the provider-inventory "provider snapshot" glossary entry. * fix(indexer): stop a corrupt chunk from poisoning the fetch cache ArchiveBlockSource#fetchChunk cached getChunk().then(onFulfilled, onRejected); a throw inside onFulfilled (a corrupt chunk line decoding to a non-object so record.height throws) rejected the cached promise without running the reset, so every later read for that range awaited the stale rejection. Switch to .then().catch() so a mapping failure also clears entry.chunkFetch and the retry re-fetches, matching the sibling fix in resolveChainId. Also hoist the duplicated buildRecord test fixture into test/fakes/build-raw-block-record so the three archive specs share one shape. --- .../.claude/skills/verify/SKILL.md | 54 ++ apps/chain-indexer/README.md | 16 + apps/chain-indexer/env/.env.sample | 2 + apps/chain-indexer/package.json | 3 +- .../src/archive/archive-block-source.spec.ts | 158 ++++++ .../src/archive/archive-block-source.ts | 145 ++++++ .../src/archive/archive-codec.spec.ts | 40 ++ .../src/archive/archive-codec.ts | 19 + .../src/archive/archive-layout.spec.ts | 63 +++ .../src/archive/archive-layout.ts | 46 ++ .../src/archive/block-archive.service.spec.ts | 203 ++++++++ .../src/archive/block-archive.service.ts | 154 ++++++ .../src/config/env.config.spec.ts | 22 + apps/chain-indexer/src/config/env.config.ts | 8 + .../pipeline/backfill-runner.service.spec.ts | 86 +++- .../src/pipeline/backfill-runner.service.ts | 26 +- .../src/pipeline/sync-runner.service.spec.ts | 128 +++++ .../src/pipeline/sync-runner.service.ts | 16 +- .../src/providers/archive.provider.spec.ts | 38 ++ .../src/providers/archive.provider.ts | 41 ++ apps/chain-indexer/src/providers/index.ts | 1 + apps/chain-indexer/src/rpc/rpc-types.ts | 3 + .../test/fakes/build-raw-block-record.ts | 15 + .../test/fakes/in-memory-object-store.ts | 57 ++ apps/provider-inventory/CONTEXT.md | 2 +- package-lock.json | 486 +++++++++++++++++- 26 files changed, 1813 insertions(+), 19 deletions(-) create mode 100644 apps/chain-indexer/.claude/skills/verify/SKILL.md create mode 100644 apps/chain-indexer/src/archive/archive-block-source.spec.ts create mode 100644 apps/chain-indexer/src/archive/archive-block-source.ts create mode 100644 apps/chain-indexer/src/archive/archive-codec.spec.ts create mode 100644 apps/chain-indexer/src/archive/archive-codec.ts create mode 100644 apps/chain-indexer/src/archive/archive-layout.spec.ts create mode 100644 apps/chain-indexer/src/archive/archive-layout.ts create mode 100644 apps/chain-indexer/src/archive/block-archive.service.spec.ts create mode 100644 apps/chain-indexer/src/archive/block-archive.service.ts create mode 100644 apps/chain-indexer/src/pipeline/sync-runner.service.spec.ts create mode 100644 apps/chain-indexer/src/providers/archive.provider.spec.ts create mode 100644 apps/chain-indexer/src/providers/archive.provider.ts create mode 100644 apps/chain-indexer/test/fakes/build-raw-block-record.ts create mode 100644 apps/chain-indexer/test/fakes/in-memory-object-store.ts diff --git a/apps/chain-indexer/.claude/skills/verify/SKILL.md b/apps/chain-indexer/.claude/skills/verify/SKILL.md new file mode 100644 index 0000000000..05e71125a4 --- /dev/null +++ b/apps/chain-indexer/.claude/skills/verify/SKILL.md @@ -0,0 +1,54 @@ +--- +name: verify +description: Build, launch, and drive apps/chain-indexer locally to verify sync/backfill/archive changes end-to-end against sandbox RPC, a scratch Postgres, and a fake-gcs-server emulator. +--- + +# Verifying chain-indexer locally + +## Build and run + +```bash +cd apps/chain-indexer +npm run build # tsup -> dist/server.js (CJS) +node dist/server.js # role picked via INDEXER_ROLE env +``` + +Pass config as process env vars; they override `env/.env*` (dotenvx does not overwrite existing env). Minimum: `NETWORK=sandbox POSTGRES_DB_URI=postgres://postgres:password@localhost:5432/`. Migrations run on boot. Sandbox RPC defaults from `@akashnetwork/net` work; the chain id is `sandbox-2` and history is unpruned from height 1. + +- `INDEXER_ROLE=sync` tails the chain (`SYNC_START_HEIGHT` to pin the start); stop it with SIGTERM (orderly shutdown, exit 0). +- `INDEXER_ROLE=backfill BACKFILL_FROM_HEIGHT=... BACKFILL_TO_HEIGHT=...` runs to completion and exits 0; interrupted runs exit 1 and resume from the per-range checkpoint on rerun. + +macOS has no `timeout`; to bound a run, spawn `node dist/server.js` from a wrapper that `child.kill("SIGTERM")`s after N ms. + +## Scratch Postgres + +Local native PG accepts `postgres://postgres:password@localhost:5432`. Create/drop a scratch DB per run: + +```bash +psql postgres://postgres:password@localhost:5432/postgres -c "CREATE DATABASE chain_indexer_e2e" +``` + +## Archive verification (GCS emulator) + +```bash +docker run -d --name fake-gcs -p 4443:4443 fsouza/fake-gcs-server -scheme http +curl -X POST "http://localhost:4443/storage/v1/b?project=test" -H "Content-Type: application/json" -d '{"name":"raw-blocks"}' +``` + +Run the indexer with `ARCHIVE_BUCKET=raw-blocks ARCHIVE_STORAGE_API_ENDPOINT=http://localhost:4443`. + +**Gotcha:** do NOT use `STORAGE_EMULATOR_HOST` — it flips @google-cloud/storage into custom-endpoint mode with unprefixed paths (`GET /b/...`); fake-gcs-server only serves `/storage/v1/...`, so saves appear to work but every download 404s and the archive silently falls back to RPC. `ARCHIVE_STORAGE_API_ENDPOINT` (the `apiEndpoint` option) keeps standard paths and both directions work. + +Inspect objects with the emulator's JSON API (`generation` proves idempotent no-rewrite): + +```bash +curl "http://localhost:4443/storage/v1/b/raw-blocks/o?prefix=sandbox-2/chunks/" +``` + +## Proving "no RPC block fetches" (AC2-style checks) + +Point `RPC_NODE_ENDPOINTS` at a local counting proxy that forwards to `https://rpc.sandbox-2.aksh.pw` and tallies paths; expose the tally on a magic route and assert `/block` and `/block_results` are absent. An archive-served 2,000-block replay does ~2 `/status` calls and finishes in ~2s (vs ~80s RPC-fed). + +## Log levels + +Per-block events (`BLOCK_COMMITTED`) are debug; `LOG_LEVEL=info` shows `SYNC_PROGRESS` only every 100 blocks. Judge sync activity by DB rows or bucket objects, not info-level log counts. diff --git a/apps/chain-indexer/README.md b/apps/chain-indexer/README.md index 14750241c0..fdfe0533f5 100644 --- a/apps/chain-indexer/README.md +++ b/apps/chain-indexer/README.md @@ -11,6 +11,10 @@ NETWORK = mainnet | sandbox | testnet Currently implemented: `sync` (live tail with per-block atomic commits and a parent-hash continuity check), `backfill` (historical catch-up over an explicit height range), and a minimal `api` (healthz + status). `jobs` exits with `ROLE_NOT_IMPLEMENTED`. +## Scope + +This app owns **chain-derived data only**: blocks, transactions, messages, on-chain provider/audit records, and the network aggregates computed from them. Off-chain provider data — pinging provider `/status` endpoints, provider inventory, uptime, IP geolocation, and the GPU breakdown derived from inventory — lives in `apps/provider-inventory`, not here. The one deliberate off-chain exception is **pricing** (AKT price history) plus **Keybase** validator identity, which the `jobs` role fetches because the daily USD aggregates and validator records need them; those enrich chain entities rather than providers. + Writers do not use leader election. Inserts are natural-keyed and conflict-ignoring and the `indexer_state` checkpoint only moves forward (`GREATEST` upsert), so overlapping writers on the same stream (e.g. two pods during a rolling deploy) duplicate work but cannot corrupt data or regress the checkpoint. Run one replica per writer role (`replicas: 1` for sync, `parallelism: 1` for backfill Jobs) to avoid the wasted work. ## Running locally @@ -48,6 +52,18 @@ BACKFILL_TO_HEIGHT=200000 Blocks are fetched from RPC in parallel (`BACKFILL_CONCURRENCY`, default 10) and committed strictly in order in batches of `BACKFILL_BATCH_SIZE` blocks (default 200), each batch in one Postgres transaction together with the checkpoint advance. Progress is checkpointed per range under the `indexer_state` stream `backfill:{from}-{to}`, so killing and restarting the job resumes at the checkpoint without gaps or duplicates, and re-running a completed range exits 0 immediately. Changing the range creates a fresh checkpoint row. All inserts are natural-keyed and conflict-ignoring, so a backfill can run against the same database as live sync, and a duplicate backfill pod on the same range is harmless. +## Raw block archive + +Set `ARCHIVE_BUCKET` to a GCS bucket name to keep a zstd-compressed copy of every raw `/block` and `/block_results` payload, so handler fixes and new modules can be replayed without re-fetching history from RPC. Leave it unset and both roles behave exactly as before (the boot log says `ARCHIVE_DISABLED`). Authentication uses Application Default Credentials; no key material is configured in the app. + +Live sync writes one staged object per block (`/blocks/.json.zst`) before the database commit, so a block is never committed without being archived. Backfill reads each height from the archive first: a 1,000-block chunk (`/chunks/-.ndjson.zst`), then a staged single, then RPC as the last resort. Any pass over a fully covered aligned range compacts it into a chunk and deletes the staged singles it consumed. There is no separate compactor: replays and backfills compact as a side effect. Ranges that cannot complete a chunk (partial edges of the run) stay as staged singles until a later full-range pass heals them. + +Two known gaps, both healable. Blocks are archived before the parent-hash continuity check, so a poisoned RPC node can pin a bad block for a height into the immutable archive even though sync halts and never commits it; a later replay of that range decodes the bad record and trips the same continuity check, which makes the divergence detectable, but the object has to be deleted by hand before a replay can archive the good copy. And a backfill killed mid-chunk loses that chunk's in-memory buffer: blocks committed before the kill are temporarily absent from the archive, and any later replay over the full range re-fetches them and compacts the chunk. + +Chunk compaction writes the chunk before deleting the staged singles it consumed. This ordering never loses data — the chunk is authoritative and reads prefer it — but a crash landing between the two calls leaves those staged singles behind, and because a later read short-circuits on the now-existing chunk the delete never runs again for that range. The leftover objects are bounded (at most one chunk's worth per crash) and cost storage only; expire them with a GCS bucket lifecycle rule on the `/blocks/` prefix rather than reordering the writes. + +Object keys are namespaced by the chain id reported by RPC `/status` (e.g. `sandbox-2/...`), so a sandbox chain reset starts a fresh namespace instead of mixing archives. For local verification against an emulator such as fake-gcs-server, point `ARCHIVE_STORAGE_API_ENDPOINT` at it (e.g. `http://localhost:4443`); the SDK's `STORAGE_EMULATOR_HOST` variable does not work here because it switches the client to request paths the emulator rejects. If the archive is unavailable, sync retries and then halts rather than committing unarchived blocks, and a backfill Job fails so the scheduler can retry it. + ## Tests ```bash diff --git a/apps/chain-indexer/env/.env.sample b/apps/chain-indexer/env/.env.sample index 78363d5847..126216bcde 100644 --- a/apps/chain-indexer/env/.env.sample +++ b/apps/chain-indexer/env/.env.sample @@ -6,6 +6,8 @@ SYNC_START_HEIGHT= SYNC_POLL_INTERVAL_MS=3000 BACKFILL_FROM_HEIGHT= BACKFILL_TO_HEIGHT= +ARCHIVE_BUCKET= +ARCHIVE_STORAGE_API_ENDPOINT= BACKFILL_CONCURRENCY=10 BACKFILL_BATCH_SIZE=200 RPC_TIMEOUT_MS=15000 diff --git a/apps/chain-indexer/package.json b/apps/chain-indexer/package.json index f340e9a936..17328b775a 100644 --- a/apps/chain-indexer/package.json +++ b/apps/chain-indexer/package.json @@ -27,6 +27,7 @@ "@cosmjs/encoding": "~0.38.0", "@cosmjs/proto-signing": "~0.38.0", "@cosmjs/stargate": "~0.38.0", + "@google-cloud/storage": "^7.21.0", "@hono/node-server": "1.13.7", "@hono/otel": "~0.4.0", "@hono/zod-openapi": "0.18.4", @@ -43,7 +44,7 @@ "devDependencies": { "@akashnetwork/dev-config": "*", "@types/lodash": "^4.17.0", - "@types/node": "^22.13.11", + "@types/node": "^22.15.0", "@typescript-eslint/eslint-plugin": "^8.64.0", "@vitest/coverage-v8": "^4.1.5", "cosmjs-types": "~0.11.0", diff --git a/apps/chain-indexer/src/archive/archive-block-source.spec.ts b/apps/chain-indexer/src/archive/archive-block-source.spec.ts new file mode 100644 index 0000000000..4871e7947a --- /dev/null +++ b/apps/chain-indexer/src/archive/archive-block-source.spec.ts @@ -0,0 +1,158 @@ +import { describe, expect, it } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import { ArchiveBlockSource } from "@src/archive/archive-block-source"; +import type { ChunkRange, RawBlockRecord } from "@src/archive/archive-layout"; +import type { BlockArchiveService } from "@src/archive/block-archive.service"; +import type { LoggerService } from "@src/providers/logging.provider"; +import type { RpcClientPool } from "@src/rpc/rpc-client-pool.service"; + +import { buildRawBlockRecord } from "@test/fakes/build-raw-block-record"; + +describe(ArchiveBlockSource.name, () => { + it("serves an archived chunk to concurrent callers with a single fetch and zero rpc calls", async () => { + const { source, archive, pool } = setup({ startHeight: 2_000, endHeight: 2_999 }); + archive.getChunk.mockResolvedValue(buildChunkRecords({ start: 2_000, end: 2_999 })); + + const records = await Promise.all(Array.from({ length: 50 }, (_, index) => source.getRecord(2_000 + index))); + + expect(records.map(record => record.height)).toEqual(Array.from({ length: 50 }, (_, index) => 2_000 + index)); + expect(archive.getChunk).toHaveBeenCalledTimes(1); + expect(pool.getBlock).not.toHaveBeenCalled(); + expect(archive.putChunkIfAbsent).not.toHaveBeenCalled(); + expect(archive.putStagedBlockIfAbsent).not.toHaveBeenCalled(); + }); + + it("prefers a staged block over rpc when the chunk misses", async () => { + const { source, archive, pool } = setup({ startHeight: 2_000, endHeight: 2_099 }); + archive.getStagedBlock.mockImplementation(async height => (height === 2_000 ? buildRawBlockRecord(2_000) : null)); + + const staged = await source.getRecord(2_000); + const fetched = await source.getRecord(2_001); + + expect(staged).toEqual(buildRawBlockRecord(2_000)); + expect(fetched.height).toBe(2_001); + expect(pool.getBlock).toHaveBeenCalledTimes(1); + expect(pool.getBlock).toHaveBeenCalledWith(2_001); + }); + + it("stages rpc-fetched blocks immediately in a range that cannot complete a chunk", async () => { + const { source, archive } = setup({ startHeight: 2_000, endHeight: 2_499 }); + + for (let height = 2_000; height <= 2_499; height++) { + await source.getRecord(height); + } + + expect(archive.putStagedBlockIfAbsent).toHaveBeenCalledTimes(500); + expect(archive.putChunkIfAbsent).not.toHaveBeenCalled(); + }); + + it("compacts a completed chunk range and deletes only the staged blocks it consumed", async () => { + const { source, archive, logger } = setup({ startHeight: 2_000, endHeight: 2_999 }); + archive.getStagedBlock.mockImplementation(async height => (height <= 2_001 ? buildRawBlockRecord(height) : null)); + + for (let height = 2_000; height <= 2_999; height++) { + await source.getRecord(height); + } + + expect(archive.putChunkIfAbsent).toHaveBeenCalledTimes(1); + const [range, records] = archive.putChunkIfAbsent.mock.calls[0]; + expect(range).toEqual({ start: 2_000, end: 2_999 }); + expect(records).toHaveLength(1_000); + expect(records[0].height).toBe(2_000); + expect(records[999].height).toBe(2_999); + expect(archive.putStagedBlockIfAbsent).not.toHaveBeenCalled(); + expect(archive.deleteStagedBlocks).toHaveBeenCalledWith([2_000, 2_001]); + expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ event: "ARCHIVE_RANGE_COMPACTED" })); + }); + + it("re-attempts the chunk put when a retried call follows a failed flush", async () => { + const { source, archive } = setup({ startHeight: 2_000, endHeight: 2_999 }); + archive.putChunkIfAbsent.mockRejectedValueOnce(new Error("gcs down")); + + for (let height = 2_000; height <= 2_998; height++) { + await source.getRecord(height); + } + await expect(source.getRecord(2_999)).rejects.toThrow("gcs down"); + await expect(source.getRecord(2_999)).resolves.toEqual(expect.objectContaining({ height: 2_999 })); + + expect(archive.putChunkIfAbsent).toHaveBeenCalledTimes(2); + }); + + it("re-issues the chunk fetch when a retried call follows a rejected fetch", async () => { + const { source, archive } = setup({ startHeight: 2_000, endHeight: 2_999 }); + archive.getChunk.mockRejectedValueOnce(new Error("gcs down")); + + await expect(source.getRecord(2_000)).rejects.toThrow("gcs down"); + await expect(source.getRecord(2_000)).resolves.toEqual(expect.objectContaining({ height: 2_000 })); + + expect(archive.getChunk).toHaveBeenCalledTimes(2); + }); + + it("re-issues the chunk fetch when a resolved chunk fails to map into records", async () => { + const { source, archive } = setup({ startHeight: 2_000, endHeight: 2_999 }); + archive.getChunk.mockResolvedValueOnce([null] as unknown as RawBlockRecord[]); + + await expect(source.getRecord(2_000)).rejects.toThrow(); + await expect(source.getRecord(2_000)).resolves.toEqual(expect.objectContaining({ height: 2_000 })); + + expect(archive.getChunk).toHaveBeenCalledTimes(2); + }); + + it("serves a buffered record on re-entry without another rpc fetch", async () => { + const { source, pool } = setup({ startHeight: 2_000, endHeight: 2_999 }); + + const first = await source.getRecord(2_000); + const second = await source.getRecord(2_000); + + expect(second).toBe(first); + expect(pool.getBlock).toHaveBeenCalledTimes(1); + }); + + it("evicts a range left behind by the ascending walk", async () => { + const { source, archive } = setup({ startHeight: 1_000, endHeight: 3_999 }); + + for (let height = 1_000; height <= 3_999; height++) { + await source.getRecord(height); + } + await source.getRecord(1_500); + + expect(archive.getChunk).toHaveBeenCalledTimes(4); + }); + + describe("when the archive is disabled", () => { + it("fetches straight from rpc without touching the archive", async () => { + const { source, archive, pool } = setup({ startHeight: 2_000, endHeight: 2_999, disabled: true }); + + const record = await source.getRecord(2_000); + + expect(record.height).toBe(2_000); + expect(pool.getBlock).toHaveBeenCalledWith(2_000); + expect(archive.getChunk).not.toHaveBeenCalled(); + expect(archive.putStagedBlockIfAbsent).not.toHaveBeenCalled(); + }); + }); + + function setup(input: { startHeight: number; endHeight: number; disabled?: boolean }) { + const archive = mock(); + archive.isEnabled.mockReturnValue(!input.disabled); + archive.getChunk.mockResolvedValue(null); + archive.getStagedBlock.mockResolvedValue(null); + archive.putChunkIfAbsent.mockResolvedValue(undefined); + archive.putStagedBlockIfAbsent.mockResolvedValue(undefined); + archive.deleteStagedBlocks.mockResolvedValue(undefined); + + const pool = mock(); + pool.getBlock.mockImplementation(async height => buildRawBlockRecord(height).block); + pool.getBlockResults.mockImplementation(async height => buildRawBlockRecord(height).block_results); + + const logger = mock(); + const source = new ArchiveBlockSource({ archive, pool, logger, startHeight: input.startHeight, endHeight: input.endHeight }); + + return { source, archive, pool, logger }; + } + + function buildChunkRecords(range: ChunkRange): RawBlockRecord[] { + return Array.from({ length: range.end - range.start + 1 }, (_, index) => buildRawBlockRecord(range.start + index)); + } +}); diff --git a/apps/chain-indexer/src/archive/archive-block-source.ts b/apps/chain-indexer/src/archive/archive-block-source.ts new file mode 100644 index 0000000000..31c15b4874 --- /dev/null +++ b/apps/chain-indexer/src/archive/archive-block-source.ts @@ -0,0 +1,145 @@ +import type { ChunkRange, RawBlockRecord } from "@src/archive/archive-layout"; +import { CHUNK_SIZE, chunkRangeFor, fetchRawBlock, isRangeContained } from "@src/archive/archive-layout"; +import type { BlockArchiveService } from "@src/archive/block-archive.service"; +import type { LoggerService } from "@src/providers/logging.provider"; +import type { RpcClientPool } from "@src/rpc/rpc-client-pool.service"; + +interface RangeEntry { + range: ChunkRange; + /** Only a range fully inside the run's bounds can ever fill a 1,000-block buffer, so partial edge ranges stage singles instead. */ + chunkEligible: boolean; + chunkFetch: Promise | null> | null; + buffer: Map; + stagedHits: Set; + flush: Promise | null; +} + +/** + * Per-run block source for the backfill runner: serves each height from the archive chunk if one + * exists, then from a staged single, then from RPC — and compacts every fully-covered chunk range + * it passes over as a side effect (put chunk, delete consumed staged singles). + * + * Failed archive calls reset their cached promise before rethrowing, so the runner's existing + * retryWithBackoff around getRecord re-attempts a fresh GET/PUT instead of awaiting a poisoned + * promise; after the runner's attempts are exhausted the Job fails (halt policy). + */ +export class ArchiveBlockSource { + readonly #archive: BlockArchiveService; + readonly #pool: RpcClientPool; + readonly #logger: LoggerService; + readonly #startHeight: number; + readonly #endHeight: number; + + /** Keyed by range start. Consumption is strictly ascending with a bounded fetch window, so at most two ranges are ever live. */ + readonly #entries = new Map(); + + constructor(params: { archive: BlockArchiveService; pool: RpcClientPool; logger: LoggerService; startHeight: number; endHeight: number }) { + this.#archive = params.archive; + this.#pool = params.pool; + this.#logger = params.logger; + this.#startHeight = params.startHeight; + this.#endHeight = params.endHeight; + } + + async getRecord(height: number): Promise { + if (!this.#archive.isEnabled()) { + return await fetchRawBlock(this.#pool, height); + } + + const entry = this.#entryFor(height); + const chunkRecord = (await this.#fetchChunk(entry))?.get(height); + if (chunkRecord) { + return chunkRecord; + } + + let record = entry.buffer.get(height); + if (!record) { + record = await this.#loadRecord(entry, height); + entry.buffer.set(height, record); + } + + if (entry.chunkEligible && entry.buffer.size === CHUNK_SIZE) { + await this.#flush(entry); + } + + return record; + } + + #entryFor(height: number): RangeEntry { + const range = chunkRangeFor(height); + let entry = this.#entries.get(range.start); + + if (!entry) { + entry = { + range, + chunkEligible: isRangeContained(range, this.#startHeight, this.#endHeight), + chunkFetch: null, + buffer: new Map(), + stagedHits: new Set(), + flush: null + }; + this.#entries.set(range.start, entry); + this.#evictBehind(range.start); + } + + return entry; + } + + #evictBehind(newStart: number): void { + for (const start of this.#entries.keys()) { + if (start <= newStart - 2 * CHUNK_SIZE) { + this.#entries.delete(start); + } + } + } + + /** Attempted for every range — a partial edge range may still be covered by a chunk from an earlier full replay. */ + #fetchChunk(entry: RangeEntry): Promise | null> { + entry.chunkFetch ??= this.#archive + .getChunk(entry.range) + .then(records => records && new Map(records.map(record => [record.height, record]))) + .catch(error => { + entry.chunkFetch = null; + throw error; + }); + return entry.chunkFetch; + } + + async #loadRecord(entry: RangeEntry, height: number): Promise { + const staged = await this.#archive.getStagedBlock(height); + if (staged) { + entry.stagedHits.add(height); + return staged; + } + + const record = await fetchRawBlock(this.#pool, height); + if (!entry.chunkEligible) { + await this.#archive.putStagedBlockIfAbsent(record); + } + return record; + } + + async #flush(entry: RangeEntry): Promise { + entry.flush ??= this.#flushChunk(entry); + try { + await entry.flush; + } catch (error) { + entry.flush = null; + throw error; + } + } + + async #flushChunk(entry: RangeEntry): Promise { + const records = [...entry.buffer.values()].sort((a, b) => a.height - b.height); + await this.#archive.putChunkIfAbsent(entry.range, records); + await this.#archive.deleteStagedBlocks([...entry.stagedHits]); + this.#logger.info({ + event: "ARCHIVE_RANGE_COMPACTED", + startHeight: entry.range.start, + endHeight: entry.range.end, + stagedConsumed: entry.stagedHits.size + }); + entry.buffer.clear(); + entry.stagedHits.clear(); + } +} diff --git a/apps/chain-indexer/src/archive/archive-codec.spec.ts b/apps/chain-indexer/src/archive/archive-codec.spec.ts new file mode 100644 index 0000000000..9c86f580c7 --- /dev/null +++ b/apps/chain-indexer/src/archive/archive-codec.spec.ts @@ -0,0 +1,40 @@ +import { zstdCompressSync, zstdDecompressSync } from "node:zlib"; +import { describe, expect, it } from "vitest"; + +import { decodeRecords, encodeRecords } from "@src/archive/archive-codec"; + +import { buildRawBlockRecord } from "@test/fakes/build-raw-block-record"; + +describe(encodeRecords.name, () => { + it("round-trips multiple records with nested and unicode content", () => { + const records = [buildRawBlockRecord(1, { memo: "héllo ✨" }), buildRawBlockRecord(2, { memo: 'line\nbreak\tand "quotes"' })]; + + expect(decodeRecords(encodeRecords(records))).toEqual(records); + }); + + it("produces a zstd frame", () => { + const encoded = encodeRecords([buildRawBlockRecord(1)]); + + expect([...encoded.subarray(0, 4)]).toEqual([0x28, 0xb5, 0x2f, 0xfd]); + }); + + it("escapes embedded newlines so every record stays on one NDJSON line", () => { + const encoded = encodeRecords([buildRawBlockRecord(1, { memo: "a\nb" }), buildRawBlockRecord(2)]); + const lines = zstdDecompressSync(encoded).toString("utf8").split("\n").filter(Boolean); + + expect(lines).toHaveLength(2); + }); +}); + +describe(decodeRecords.name, () => { + it("tolerates a trailing newline", () => { + const record = buildRawBlockRecord(7); + const buffer = zstdCompressSync(Buffer.from(`${JSON.stringify(record)}\n`)); + + expect(decodeRecords(buffer)).toEqual([record]); + }); + + it("throws on a buffer that is not zstd", () => { + expect(() => decodeRecords(Buffer.from("not zstd at all"))).toThrow(); + }); +}); diff --git a/apps/chain-indexer/src/archive/archive-codec.ts b/apps/chain-indexer/src/archive/archive-codec.ts new file mode 100644 index 0000000000..6bd6ccb745 --- /dev/null +++ b/apps/chain-indexer/src/archive/archive-codec.ts @@ -0,0 +1,19 @@ +import { constants, zstdCompressSync, zstdDecompressSync } from "node:zlib"; + +import type { RawBlockRecord } from "@src/archive/archive-layout"; + +/** Level 3 is zstd's default: ~3-4x compression on block JSON at negligible CPU next to RPC latency. */ +const ZSTD_LEVEL = 3; + +export function encodeRecords(records: RawBlockRecord[]): Buffer { + const ndjson = records.map(record => JSON.stringify(record)).join("\n"); + return zstdCompressSync(Buffer.from(`${ndjson}\n`), { params: { [constants.ZSTD_c_compressionLevel]: ZSTD_LEVEL } }); +} + +export function decodeRecords(buffer: Buffer): RawBlockRecord[] { + return zstdDecompressSync(buffer) + .toString("utf8") + .split("\n") + .filter(line => line.length > 0) + .map(line => JSON.parse(line) as RawBlockRecord); +} diff --git a/apps/chain-indexer/src/archive/archive-layout.spec.ts b/apps/chain-indexer/src/archive/archive-layout.spec.ts new file mode 100644 index 0000000000..301290cac5 --- /dev/null +++ b/apps/chain-indexer/src/archive/archive-layout.spec.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from "vitest"; + +import { chunkKey, chunkRangeFor, isRangeContained, stagedBlockKey } from "@src/archive/archive-layout"; + +describe(chunkRangeFor.name, () => { + it("maps height 1 into the first chunk range", () => { + expect(chunkRangeFor(1)).toEqual({ start: 0, end: 999 }); + }); + + it("maps height 999 into the first chunk range", () => { + expect(chunkRangeFor(999)).toEqual({ start: 0, end: 999 }); + }); + + it("maps height 1000 into the second chunk range", () => { + expect(chunkRangeFor(1_000)).toEqual({ start: 1_000, end: 1_999 }); + }); + + it("maps height 1999 into the second chunk range", () => { + expect(chunkRangeFor(1_999)).toEqual({ start: 1_000, end: 1_999 }); + }); + + it("maps a large height into its aligned chunk range", () => { + expect(chunkRangeFor(23_456_789)).toEqual({ start: 23_456_000, end: 23_456_999 }); + }); +}); + +describe(isRangeContained.name, () => { + it("contains a range exactly matching the bounds", () => { + expect(isRangeContained({ start: 2_000, end: 2_999 }, 2_000, 2_999)).toBe(true); + }); + + it("contains a range strictly inside the bounds", () => { + expect(isRangeContained({ start: 2_000, end: 2_999 }, 1_500, 3_500)).toBe(true); + }); + + it("does not contain a range whose start is below the lower bound", () => { + expect(isRangeContained({ start: 2_000, end: 2_999 }, 2_500, 3_500)).toBe(false); + }); + + it("does not contain a range whose end is above the upper bound", () => { + expect(isRangeContained({ start: 2_000, end: 2_999 }, 1_500, 2_500)).toBe(false); + }); + + it("does not contain the first chunk range when heights start at 1", () => { + expect(isRangeContained({ start: 0, end: 999 }, 1, 5_000)).toBe(false); + }); +}); + +describe(chunkKey.name, () => { + it("builds a zero-padded chunk key under the chain id", () => { + expect(chunkKey("sandbox-01", { start: 2_000, end: 2_999 })).toBe("sandbox-01/chunks/0000002000-0000002999.ndjson.zst"); + }); + + it("keeps keys sortable for heights above ten digits of padding", () => { + expect(chunkKey("akashnet-2", { start: 23_456_000, end: 23_456_999 })).toBe("akashnet-2/chunks/0023456000-0023456999.ndjson.zst"); + }); +}); + +describe(stagedBlockKey.name, () => { + it("builds a zero-padded staged block key under the chain id", () => { + expect(stagedBlockKey("sandbox-01", 1_234)).toBe("sandbox-01/blocks/0000001234.json.zst"); + }); +}); diff --git a/apps/chain-indexer/src/archive/archive-layout.ts b/apps/chain-indexer/src/archive/archive-layout.ts new file mode 100644 index 0000000000..3d74b629c1 --- /dev/null +++ b/apps/chain-indexer/src/archive/archive-layout.ts @@ -0,0 +1,46 @@ +import type { RpcClientPool } from "@src/rpc/rpc-client-pool.service"; +import type { RpcBlockResult, RpcBlockResultsResult } from "@src/rpc/rpc-types"; + +export const CHUNK_SIZE = 1_000; + +/** Padding width for heights in object keys; 10 digits keeps keys lexicographically sortable past height 9,999,999,999. */ +const HEIGHT_PAD_WIDTH = 10; + +/** One archived block: the verbatim raw RPC payloads of /block and /block_results plus the height they belong to. */ +export interface RawBlockRecord { + height: number; + block: RpcBlockResult; + block_results: RpcBlockResultsResult; +} + +/** Fetches the raw /block and /block_results payloads for a height in parallel and pairs them into one record. */ +export async function fetchRawBlock(pool: RpcClientPool, height: number): Promise { + const [block, blockResults] = await Promise.all([pool.getBlock(height), pool.getBlockResults(height)]); + return { height, block, block_results: blockResults }; +} + +export interface ChunkRange { + start: number; + end: number; +} + +export function chunkRangeFor(height: number): ChunkRange { + const start = Math.floor(height / CHUNK_SIZE) * CHUNK_SIZE; + return { start, end: start + CHUNK_SIZE - 1 }; +} + +export function isRangeContained(range: ChunkRange, startHeight: number, endHeight: number): boolean { + return range.start >= startHeight && range.end <= endHeight; +} + +export function chunkKey(chainId: string, range: ChunkRange): string { + return `${chainId}/chunks/${padHeight(range.start)}-${padHeight(range.end)}.ndjson.zst`; +} + +export function stagedBlockKey(chainId: string, height: number): string { + return `${chainId}/blocks/${padHeight(height)}.json.zst`; +} + +function padHeight(height: number): string { + return String(height).padStart(HEIGHT_PAD_WIDTH, "0"); +} diff --git a/apps/chain-indexer/src/archive/block-archive.service.spec.ts b/apps/chain-indexer/src/archive/block-archive.service.spec.ts new file mode 100644 index 0000000000..f768063cfc --- /dev/null +++ b/apps/chain-indexer/src/archive/block-archive.service.spec.ts @@ -0,0 +1,203 @@ +import { describe, expect, it } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import type { RawBlockRecord } from "@src/archive/archive-layout"; +import { BlockArchiveService } from "@src/archive/block-archive.service"; +import { envSchema } from "@src/config/env.config"; +import type { LoggerService } from "@src/providers/logging.provider"; +import type { RpcClientPool } from "@src/rpc/rpc-client-pool.service"; +import type { RpcStatusResult } from "@src/rpc/rpc-types"; + +import { buildRawBlockRecord } from "@test/fakes/build-raw-block-record"; +import { httpError, InMemoryObjectStore } from "@test/fakes/in-memory-object-store"; + +describe(BlockArchiveService.name, () => { + it("reports enabled when constructed with a store", () => { + const { service } = setup(); + + expect(service.isEnabled()).toBe(true); + }); + + it("writes staged blocks under the zero-padded per-chain key", async () => { + const { service, store } = setup(); + + await service.putStagedBlockIfAbsent(buildRawBlockRecord(1_234)); + + expect([...store.objects.keys()]).toEqual(["raw-blocks/sandbox-01/blocks/0000001234.json.zst"]); + }); + + it("writes chunks under the zero-padded per-chain range key", async () => { + const { service, store } = setup(); + + await service.putChunkIfAbsent({ start: 2_000, end: 2_999 }, buildChunkRecords(2_000, 2_999)); + + expect([...store.objects.keys()]).toEqual(["raw-blocks/sandbox-01/chunks/0000002000-0000002999.ndjson.zst"]); + }); + + it("rejects a chunk put whose records do not cover the range", async () => { + const { service, store } = setup(); + + await expect(service.putChunkIfAbsent({ start: 2_000, end: 2_999 }, [buildRawBlockRecord(2_000)])).rejects.toThrow("1000 records"); + + expect(store.objects.size).toBe(0); + }); + + it("round-trips a staged block", async () => { + const { service } = setup(); + const record = buildRawBlockRecord(42); + + await service.putStagedBlockIfAbsent(record); + + await expect(service.getStagedBlock(42)).resolves.toEqual(record); + }); + + it("round-trips a chunk", async () => { + const { service } = setup(); + const records = buildChunkRecords(2_000, 2_999); + + await service.putChunkIfAbsent({ start: 2_000, end: 2_999 }, records); + + await expect(service.getChunk({ start: 2_000, end: 2_999 })).resolves.toEqual(records); + }); + + it("logs the bucket when enabled", () => { + const { service, logger } = setup(); + + service.logState(); + + expect(logger.info).toHaveBeenCalledWith({ event: "ARCHIVE_ENABLED", bucket: "raw-blocks" }); + }); + + it("keeps the original object when the same staged block is put twice", async () => { + const { service, store } = setup(); + const original = buildRawBlockRecord(42); + const replay = { ...buildRawBlockRecord(42), block_results: { height: "42", txs_results: [{ code: 1 }] } }; + + await service.putStagedBlockIfAbsent(original); + await expect(service.putStagedBlockIfAbsent(replay)).resolves.toBeUndefined(); + + await expect(service.getStagedBlock(42)).resolves.toEqual(original); + expect(store.objects.size).toBe(1); + }); + + it("returns null for a missing chunk", async () => { + const { service } = setup(); + + await expect(service.getChunk({ start: 0, end: 999 })).resolves.toBeNull(); + }); + + it("returns null for a missing staged block", async () => { + const { service } = setup(); + + await expect(service.getStagedBlock(7)).resolves.toBeNull(); + }); + + it("propagates non-404 download errors", async () => { + const { service, store } = setup(); + store.failNextDownloadWith = httpError(500, "backend blew up"); + + await expect(service.getChunk({ start: 0, end: 999 })).rejects.toThrow("backend blew up"); + }); + + it("propagates non-412 save errors", async () => { + const { service, store } = setup(); + store.failNextSaveWith = httpError(503, "unavailable"); + + await expect(service.putStagedBlockIfAbsent(buildRawBlockRecord(1))).rejects.toThrow("unavailable"); + }); + + it("deletes staged blocks best-effort without throwing on failures", async () => { + const { service, store, logger } = setup(); + await service.putStagedBlockIfAbsent(buildRawBlockRecord(1)); + await service.putStagedBlockIfAbsent(buildRawBlockRecord(2)); + store.failNextDeleteWith = httpError(503, "unavailable"); + + await expect(service.deleteStagedBlocks([1, 2, 3])).resolves.toBeUndefined(); + + expect(store.objects.size).toBe(1); + expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ event: "ARCHIVE_STAGED_DELETE_FAILED" })); + }); + + it("skips staged deletes when the chain id cannot be resolved", async () => { + const { service, pool, logger } = setup(); + pool.getStatus.mockRejectedValue(new Error("rpc down")); + + await expect(service.deleteStagedBlocks([1])).resolves.toBeUndefined(); + + expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ event: "ARCHIVE_STAGED_DELETE_FAILED", reason: "chain id unavailable" })); + }); + + it("fetches the chain id once across concurrent puts", async () => { + const { service, pool } = setup(); + + await Promise.all([service.putStagedBlockIfAbsent(buildRawBlockRecord(1)), service.putStagedBlockIfAbsent(buildRawBlockRecord(2))]); + + expect(pool.getStatus).toHaveBeenCalledTimes(1); + }); + + it("refetches the chain id after a failed status call", async () => { + const { service, pool } = setup(); + pool.getStatus.mockRejectedValueOnce(new Error("rpc down")); + + await expect(service.putStagedBlockIfAbsent(buildRawBlockRecord(1))).rejects.toThrow("rpc down"); + await expect(service.putStagedBlockIfAbsent(buildRawBlockRecord(1))).resolves.toBeUndefined(); + + expect(pool.getStatus).toHaveBeenCalledTimes(2); + }); + + it("refetches the chain id after a malformed status response poisons the first call", async () => { + const { service, pool } = setup(); + pool.getStatus.mockResolvedValueOnce({ sync_info: { latest_block_height: "1" } } as unknown as RpcStatusResult); + + await expect(service.putStagedBlockIfAbsent(buildRawBlockRecord(1))).rejects.toThrow(); + await expect(service.putStagedBlockIfAbsent(buildRawBlockRecord(1))).resolves.toBeUndefined(); + + expect(pool.getStatus).toHaveBeenCalledTimes(2); + }); + + describe("when the archive is disabled", () => { + it("reports disabled", () => { + const { service } = setup({ disabled: true }); + + expect(service.isEnabled()).toBe(false); + }); + + it("rejects reads and writes", async () => { + const { service } = setup({ disabled: true }); + + await expect(service.getChunk({ start: 0, end: 999 })).rejects.toThrow("disabled"); + await expect(service.putStagedBlockIfAbsent(buildRawBlockRecord(1))).rejects.toThrow("disabled"); + }); + + it("resolves staged deletes as a no-op", async () => { + const { service } = setup({ disabled: true }); + + await expect(service.deleteStagedBlocks([1])).resolves.toBeUndefined(); + }); + + it("logs the disabled state", () => { + const { service, logger } = setup({ disabled: true }); + + service.logState(); + + expect(logger.info).toHaveBeenCalledWith({ event: "ARCHIVE_DISABLED" }); + }); + }); + + function setup(input?: { disabled?: boolean }) { + const store = new InMemoryObjectStore(); + const config = envSchema.parse({ + POSTGRES_DB_URI: "postgres://unit:unit@localhost:5432/unit", + ARCHIVE_BUCKET: input?.disabled ? "" : "raw-blocks" + }); + const pool = mock(); + pool.getStatus.mockResolvedValue({ node_info: { network: "sandbox-01" }, sync_info: { latest_block_height: "1" } }); + const logger = mock(); + const service = new BlockArchiveService(input?.disabled ? null : store, config, pool, logger); + return { service, store, pool, logger }; + } + + function buildChunkRecords(fromHeight: number, toHeight: number): RawBlockRecord[] { + return Array.from({ length: toHeight - fromHeight + 1 }, (_, index) => buildRawBlockRecord(fromHeight + index)); + } +}); diff --git a/apps/chain-indexer/src/archive/block-archive.service.ts b/apps/chain-indexer/src/archive/block-archive.service.ts new file mode 100644 index 0000000000..ac8ab63b8a --- /dev/null +++ b/apps/chain-indexer/src/archive/block-archive.service.ts @@ -0,0 +1,154 @@ +import { inject, singleton } from "tsyringe"; + +import { decodeRecords, encodeRecords } from "@src/archive/archive-codec"; +import type { ChunkRange, RawBlockRecord } from "@src/archive/archive-layout"; +import { chunkKey, stagedBlockKey } from "@src/archive/archive-layout"; +import type { EnvConfig } from "@src/config/env.config"; +import { APP_CONFIG } from "@src/providers/app-config.provider"; +import type { ArchiveObjectStore } from "@src/providers/archive.provider"; +import { ARCHIVE_STORAGE } from "@src/providers/archive.provider"; +import { LoggerService } from "@src/providers/logging.provider"; +import { RpcClientPool } from "@src/rpc/rpc-client-pool.service"; + +const DELETE_BATCH_SIZE = 25; +const SAVE_OPTIONS = { resumable: false, contentType: "application/zstd", preconditionOpts: { ifGenerationMatch: 0 } }; + +@singleton() +export class BlockArchiveService { + readonly #storage: ArchiveObjectStore | null; + readonly #config: EnvConfig; + readonly #pool: RpcClientPool; + readonly #logger: LoggerService; + + #chainId: Promise | null = null; + + constructor( + @inject(ARCHIVE_STORAGE) storage: ArchiveObjectStore | null, + @inject(APP_CONFIG) config: EnvConfig, + @inject(RpcClientPool) pool: RpcClientPool, + @inject(LoggerService) logger: LoggerService + ) { + this.#storage = storage; + this.#config = config; + this.#pool = pool; + this.#logger = logger; + this.#logger.setContext("ARCHIVE"); + } + + isEnabled(): boolean { + return this.#storage !== null; + } + + logState(): void { + if (this.isEnabled()) { + this.#logger.info({ event: "ARCHIVE_ENABLED", bucket: this.#config.ARCHIVE_BUCKET }); + } else { + this.#logger.info({ event: "ARCHIVE_DISABLED" }); + } + } + + async getChunk(range: ChunkRange): Promise { + const buffer = await this.#download(chunkKey(await this.#resolveChainId(), range)); + return buffer ? decodeRecords(buffer) : null; + } + + async getStagedBlock(height: number): Promise { + const buffer = await this.#download(stagedBlockKey(await this.#resolveChainId(), height)); + return buffer ? decodeRecords(buffer)[0] ?? null : null; + } + + /** + * A chunk's existence short-circuits staged reads and triggers staged-single deletion, so a + * partial chunk could destroy the only copy of blocks; the record count makes that impossible. + */ + async putChunkIfAbsent(range: ChunkRange, records: RawBlockRecord[]): Promise { + const expectedCount = range.end - range.start + 1; + if (records.length !== expectedCount) { + throw new Error(`Chunk ${range.start}-${range.end} requires ${expectedCount} records, got ${records.length}`); + } + await this.#saveIfAbsent(chunkKey(await this.#resolveChainId(), range), encodeRecords(records)); + } + + async putStagedBlockIfAbsent(record: RawBlockRecord): Promise { + await this.#saveIfAbsent(stagedBlockKey(await this.#resolveChainId(), record.height), encodeRecords([record])); + } + + async deleteStagedBlocks(heights: number[]): Promise { + if (!this.isEnabled() || heights.length === 0) { + return; + } + + const chainId = await this.#resolveChainId().catch(() => null); + if (chainId === null) { + this.#logger.warn({ + event: "ARCHIVE_STAGED_DELETE_FAILED", + reason: "chain id unavailable", + heightCount: heights.length, + firstHeight: heights[0], + lastHeight: heights[heights.length - 1] + }); + return; + } + + for (let offset = 0; offset < heights.length; offset += DELETE_BATCH_SIZE) { + const batch = heights.slice(offset, offset + DELETE_BATCH_SIZE); + const outcomes = await Promise.allSettled(batch.map(height => this.#file(stagedBlockKey(chainId, height)).delete({ ignoreNotFound: true }))); + outcomes.forEach((outcome, index) => { + if (outcome.status === "rejected") { + this.#logger.warn({ event: "ARCHIVE_STAGED_DELETE_FAILED", height: batch[index], error: outcome.reason }); + } + }); + } + } + + async #saveIfAbsent(key: string, data: Buffer): Promise { + try { + await this.#file(key).save(data, SAVE_OPTIONS); + } catch (error) { + if (statusCodeOf(error) !== 412) { + throw error; + } + this.#logger.debug({ event: "ARCHIVE_OBJECT_EXISTS", key }); + } + } + + async #download(key: string): Promise { + try { + const [buffer] = await this.#file(key).download(); + return buffer; + } catch (error) { + if (statusCodeOf(error) !== 404) { + throw error; + } + return null; + } + } + + #file(key: string): ReturnType["file"]> { + if (!this.#storage || !this.#config.ARCHIVE_BUCKET) { + throw new Error("Raw block archive is disabled; check isEnabled() before calling the archive"); + } + return this.#storage.bucket(this.#config.ARCHIVE_BUCKET).file(key); + } + + /** + * The chain id (e.g. sandbox-01) namespaces every object key so a chain reset cannot mix + * archives. Concurrent callers share one in-flight /status request; a rejected fetch clears + * the cache so the caller's retry re-fetches instead of failing forever. + */ + #resolveChainId(): Promise { + this.#chainId ??= this.#pool + .getStatus() + .then(status => status.node_info.network) + .catch(error => { + this.#chainId = null; + throw error; + }); + return this.#chainId; + } +} + +function statusCodeOf(error: unknown): number | null { + const code = (error as { code?: unknown } | null)?.code; + return typeof code === "number" ? code : null; +} diff --git a/apps/chain-indexer/src/config/env.config.spec.ts b/apps/chain-indexer/src/config/env.config.spec.ts index cbafcbfc98..f448a6e043 100644 --- a/apps/chain-indexer/src/config/env.config.spec.ts +++ b/apps/chain-indexer/src/config/env.config.spec.ts @@ -52,6 +52,28 @@ describe("envSchema", () => { }); }); + it("treats an empty ARCHIVE_BUCKET as absent", () => { + const config = setup({ ARCHIVE_BUCKET: "" }); + + expect(config.ARCHIVE_BUCKET).toBeUndefined(); + }); + + it("parses ARCHIVE_BUCKET when set", () => { + const config = setup({ ARCHIVE_BUCKET: "raw-blocks" }); + + expect(config.ARCHIVE_BUCKET).toBe("raw-blocks"); + }); + + it("treats an empty ARCHIVE_STORAGE_API_ENDPOINT as absent", () => { + const config = setup({ ARCHIVE_STORAGE_API_ENDPOINT: "" }); + + expect(config.ARCHIVE_STORAGE_API_ENDPOINT).toBeUndefined(); + }); + + it("rejects a malformed ARCHIVE_STORAGE_API_ENDPOINT", () => { + expect(() => setup({ ARCHIVE_STORAGE_API_ENDPOINT: "not a url" })).toThrow(); + }); + it("does not require backfill heights for other roles", () => { const config = setup({ INDEXER_ROLE: "api", BACKFILL_FROM_HEIGHT: "", BACKFILL_TO_HEIGHT: "" }); diff --git a/apps/chain-indexer/src/config/env.config.ts b/apps/chain-indexer/src/config/env.config.ts index 1c00840be0..4915911c95 100644 --- a/apps/chain-indexer/src/config/env.config.ts +++ b/apps/chain-indexer/src/config/env.config.ts @@ -22,6 +22,14 @@ const rawEnvSchema = z.object({ BACKFILL_CONCURRENCY: z.number({ coerce: true }).int().min(1).max(64).default(10), /** How many blocks the backfill commits per Postgres transaction. */ BACKFILL_BATCH_SIZE: z.number({ coerce: true }).int().min(1).max(1_000).default(200), + /** GCS bucket for the raw block archive. Unset disables archiving entirely (sync skips appends, backfill reads straight from RPC). */ + ARCHIVE_BUCKET: z.preprocess(emptyStringAsUndefined, z.string().optional()), + /** + * Overrides the GCS API endpoint for local emulators (e.g. fake-gcs-server). The SDK's own + * STORAGE_EMULATOR_HOST is not honored here: it switches the SDK to unprefixed request paths + * that fake-gcs-server rejects, while the apiEndpoint option keeps standard JSON API paths. + */ + ARCHIVE_STORAGE_API_ENDPOINT: z.preprocess(emptyStringAsUndefined, z.string().url().optional()), /** Decoded message bodies above this serialized size are stored as null to keep pathological messages out of Postgres. */ MESSAGE_BODY_MAX_BYTES: z.number({ coerce: true }).int().positive().default(65_536), DRIZZLE_MIGRATIONS_FOLDER: z.string().default("./drizzle"), diff --git a/apps/chain-indexer/src/pipeline/backfill-runner.service.spec.ts b/apps/chain-indexer/src/pipeline/backfill-runner.service.spec.ts index 9d81537c25..85870718fc 100644 --- a/apps/chain-indexer/src/pipeline/backfill-runner.service.spec.ts +++ b/apps/chain-indexer/src/pipeline/backfill-runner.service.spec.ts @@ -2,6 +2,8 @@ import { setTimeout as delay } from "node:timers/promises"; import { describe, expect, it, vi } from "vitest"; import { mock } from "vitest-mock-extended"; +import type { RawBlockRecord } from "@src/archive/archive-layout"; +import type { BlockArchiveService } from "@src/archive/block-archive.service"; import { envSchema } from "@src/config/env.config"; import { Blocks, IndexerState } from "@src/db/schema"; import { BackfillRunnerService } from "@src/pipeline/backfill-runner.service"; @@ -149,6 +151,58 @@ describe(BackfillRunnerService.name, () => { expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ event: "BACKFILL_COMPLETED" })); }); + describe("when the archive is enabled", () => { + it("serves an archived range without rpc fetches or archive writes", async () => { + const { runner, committer, pool, archive } = setup({ fromHeight: 1_000, toHeight: 1_999, tipHeight: 10_000, archiveEnabled: true }); + archive.getChunk.mockResolvedValue(buildRawRecords(1_000, 1_999)); + + await runner.start(); + + expect(committedHeights(committer).flat()).toHaveLength(1_000); + expect(pool.getBlock).not.toHaveBeenCalled(); + expect(pool.getBlockResults).not.toHaveBeenCalled(); + expect(archive.putChunkIfAbsent).not.toHaveBeenCalled(); + expect(archive.putStagedBlockIfAbsent).not.toHaveBeenCalled(); + }); + + it("compacts an rpc-fed aligned range into a single chunk", async () => { + const { runner, committer, archive } = setup({ fromHeight: 1_000, toHeight: 1_999, tipHeight: 10_000, archiveEnabled: true }); + + await runner.start(); + + expect(committedHeights(committer).flat()).toHaveLength(1_000); + expect(archive.putChunkIfAbsent).toHaveBeenCalledTimes(1); + expect(archive.putStagedBlockIfAbsent).not.toHaveBeenCalled(); + }); + + it("stages singles for a range that cannot complete a chunk", async () => { + const { runner, archive } = setup({ fromHeight: 1_000, toHeight: 1_499, tipHeight: 10_000, archiveEnabled: true }); + + await runner.start(); + + expect(archive.putStagedBlockIfAbsent).toHaveBeenCalledTimes(500); + expect(archive.putChunkIfAbsent).not.toHaveBeenCalled(); + }); + + it("fails the job without completing when the chunk flush keeps failing", async () => { + vi.useFakeTimers(); + + try { + const { runner, archive, logger } = setup({ fromHeight: 1_000, toHeight: 1_999, tipHeight: 10_000, archiveEnabled: true }); + archive.putChunkIfAbsent.mockRejectedValue(new Error("gcs down")); + + const started = runner.start(); + started.catch(() => undefined); + await vi.runAllTimersAsync(); + + await expect(started).rejects.toThrow("gcs down"); + expect(logger.info).not.toHaveBeenCalledWith(expect.objectContaining({ event: "BACKFILL_COMPLETED" })); + } finally { + vi.useRealTimers(); + } + }); + }); + function setup(input: { fromHeight: number; toHeight: number; @@ -161,6 +215,7 @@ describe(BackfillRunnerService.name, () => { failFetchOnceAtHeight?: number; brokenParentAtHeight?: number; txCountPerBlock?: number; + archiveEnabled?: boolean; }) { const config = envSchema.parse({ POSTGRES_DB_URI: "postgres://unit:unit@localhost:5432/unit", @@ -168,7 +223,8 @@ describe(BackfillRunnerService.name, () => { BACKFILL_FROM_HEIGHT: String(input.fromHeight), BACKFILL_TO_HEIGHT: String(input.toHeight), BACKFILL_BATCH_SIZE: String(input.batchSize ?? 200), - BACKFILL_CONCURRENCY: String(input.concurrency ?? 10) + BACKFILL_CONCURRENCY: String(input.concurrency ?? 10), + ARCHIVE_BUCKET: input.archiveEnabled ? "raw-blocks" : "" }); const dbFake = { @@ -200,7 +256,10 @@ describe(BackfillRunnerService.name, () => { activeFetches++; maxActiveFetches = Math.max(maxActiveFetches, activeFetches); - await delay(input.fetchDelayMs?.(height) ?? 0); + const fetchDelayMs = input.fetchDelayMs?.(height) ?? 0; + if (fetchDelayMs > 0) { + await delay(fetchDelayMs); + } activeFetches--; return { block: { header: { height: String(height) } } } as RpcBlockResult; }); @@ -218,11 +277,19 @@ describe(BackfillRunnerService.name, () => { const committer = mock(); committer.commitBatch.mockResolvedValue(undefined); + const archive = mock(); + archive.isEnabled.mockReturnValue(input.archiveEnabled ?? false); + archive.getChunk.mockResolvedValue(null); + archive.getStagedBlock.mockResolvedValue(null); + archive.putChunkIfAbsent.mockResolvedValue(undefined); + archive.putStagedBlockIfAbsent.mockResolvedValue(undefined); + archive.deleteStagedBlocks.mockResolvedValue(undefined); + const logger = mock(); - const runner = new BackfillRunnerService(dbFake as unknown as ChainDatabase, pool, decoder, committer, config, logger); + const runner = new BackfillRunnerService(dbFake as unknown as ChainDatabase, pool, decoder, committer, archive, config, logger); - return { runner, committer, pool, logger, maxObservedConcurrency: () => maxActiveFetches }; + return { runner, committer, pool, archive, logger, maxObservedConcurrency: () => maxActiveFetches }; } function committedHeights(committer: { commitBatch: { mock: { calls: unknown[][] } } }) { @@ -251,4 +318,15 @@ describe(BackfillRunnerService.name, () => { function heightHash(height: number): Buffer { return Buffer.from(`hash-${height}`); } + + function buildRawRecords(fromHeight: number, toHeight: number): RawBlockRecord[] { + return Array.from({ length: toHeight - fromHeight + 1 }, (_, index) => { + const height = fromHeight + index; + return { + height, + block: { block: { header: { height: String(height) } } } as RpcBlockResult, + block_results: { height: String(height), txs_results: null } + }; + }); + } }); diff --git a/apps/chain-indexer/src/pipeline/backfill-runner.service.ts b/apps/chain-indexer/src/pipeline/backfill-runner.service.ts index 1961726277..68ec3e1c1a 100644 --- a/apps/chain-indexer/src/pipeline/backfill-runner.service.ts +++ b/apps/chain-indexer/src/pipeline/backfill-runner.service.ts @@ -1,6 +1,8 @@ import { eq } from "drizzle-orm"; import { inject, singleton } from "tsyringe"; +import { ArchiveBlockSource } from "@src/archive/archive-block-source"; +import { BlockArchiveService } from "@src/archive/block-archive.service"; import type { EnvConfig } from "@src/config/env.config"; import { Blocks, IndexerState } from "@src/db/schema"; import { retryWithBackoff } from "@src/lib/retry-with-backoff/retry-with-backoff"; @@ -26,6 +28,7 @@ export class BackfillRunnerService { readonly #pool: RpcClientPool; readonly #decoder: BlockDecoderService; readonly #committer: BlockCommitterService; + readonly #archive: BlockArchiveService; readonly #config: EnvConfig; readonly #logger: LoggerService; @@ -37,6 +40,7 @@ export class BackfillRunnerService { @inject(RpcClientPool) pool: RpcClientPool, @inject(BlockDecoderService) decoder: BlockDecoderService, @inject(BlockCommitterService) committer: BlockCommitterService, + @inject(BlockArchiveService) archive: BlockArchiveService, @inject(APP_CONFIG) config: EnvConfig, @inject(LoggerService) logger: LoggerService ) { @@ -44,6 +48,7 @@ export class BackfillRunnerService { this.#pool = pool; this.#decoder = decoder; this.#committer = committer; + this.#archive = archive; this.#config = config; this.#logger = logger; this.#logger.setContext("BACKFILL"); @@ -96,8 +101,17 @@ export class BackfillRunnerService { await this.#seedContinuityHash(plan.startHeight, checkpointHeight !== null); this.#logger.info({ event: "BACKFILL_STARTED", network: this.#config.NETWORK, stream, startHeight: plan.startHeight, endHeight: plan.endHeight }); + this.#archive.logState(); + + const source = new ArchiveBlockSource({ + archive: this.#archive, + pool: this.#pool, + logger: this.#logger, + startHeight: plan.startHeight, + endHeight: plan.endHeight + }); - return await this.#backfillRange(plan.startHeight, plan.endHeight, stream); + return await this.#backfillRange(plan.startHeight, plan.endHeight, stream, source); } /** @@ -111,7 +125,7 @@ export class BackfillRunnerService { * rather than the stopped flag, so a shutdown landing during the final commit still reports the * range as done instead of failing the Job for a spurious retry. */ - async #backfillRange(startHeight: number, endHeight: number, stream: string): Promise { + async #backfillRange(startHeight: number, endHeight: number, stream: string, source: ArchiveBlockSource): Promise { const startedAt = Date.now(); const inflight = new Map>(); let fetchHead = startHeight; @@ -123,7 +137,7 @@ export class BackfillRunnerService { const fillFetchWindow = () => { while (fetchHead <= endHeight && inflight.size < this.#config.BACKFILL_CONCURRENCY) { const height = fetchHead; - const prefetched = this.#fetchAndDecode(height); + const prefetched = this.#fetchAndDecode(height, source); prefetched.catch(() => undefined); inflight.set(height, prefetched); fetchHead++; @@ -180,11 +194,11 @@ export class BackfillRunnerService { } /** A pool AggregateError means every RPC endpoint already failed once, so retries back off before another full sweep. */ - async #fetchAndDecode(height: number): Promise { + async #fetchAndDecode(height: number, source: ArchiveBlockSource): Promise { return await retryWithBackoff( async () => { - const [block, blockResults] = await Promise.all([this.#pool.getBlock(height), this.#pool.getBlockResults(height)]); - return this.#decoder.decode(block, blockResults); + const record = await source.getRecord(height); + return this.#decoder.decode(record.block, record.block_results); }, { maxAttempts: FETCH_RETRY_MAX_ATTEMPTS, diff --git a/apps/chain-indexer/src/pipeline/sync-runner.service.spec.ts b/apps/chain-indexer/src/pipeline/sync-runner.service.spec.ts new file mode 100644 index 0000000000..c8a446ce86 --- /dev/null +++ b/apps/chain-indexer/src/pipeline/sync-runner.service.spec.ts @@ -0,0 +1,128 @@ +import { describe, expect, it, vi } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import type { BlockArchiveService } from "@src/archive/block-archive.service"; +import { envSchema } from "@src/config/env.config"; +import type { BlockCommitterService } from "@src/pipeline/block-committer.service"; +import type { BlockDecoderService } from "@src/pipeline/block-decoder.service"; +import type { DecodedBlock } from "@src/pipeline/decoded-block"; +import { SyncRunnerService } from "@src/pipeline/sync-runner.service"; +import type { ChainDatabase } from "@src/providers/db.provider"; +import type { LoggerService } from "@src/providers/logging.provider"; +import type { RpcClientPool } from "@src/rpc/rpc-client-pool.service"; +import type { RpcBlockResult } from "@src/rpc/rpc-types"; + +describe(SyncRunnerService.name, () => { + it("stages every synced block in the archive with its raw payloads", async () => { + const { runner, archive } = setup({ tipHeight: 3 }); + + await runner.start(); + + expect(archive.putStagedBlockIfAbsent).toHaveBeenCalledTimes(3); + expect(archive.putStagedBlockIfAbsent).toHaveBeenNthCalledWith(2, { + height: 2, + block: expect.objectContaining({ block: expect.objectContaining({ header: expect.objectContaining({ height: "2" }) }) }), + block_results: expect.objectContaining({ height: "2" }) + }); + }); + + it("archives a block before committing it", async () => { + const { runner, archive, committer } = setup({ tipHeight: 1 }); + + await runner.start(); + + expect(archive.putStagedBlockIfAbsent.mock.invocationCallOrder[0]).toBeLessThan(committer.commit.mock.invocationCallOrder[0]); + }); + + it("logs the archive state once at startup", async () => { + const { runner, archive } = setup({ tipHeight: 1 }); + + await runner.start(); + + expect(archive.logState).toHaveBeenCalledTimes(1); + }); + + it("rejects after exhausting retries when the archive stays unavailable and never commits", async () => { + vi.useFakeTimers(); + + try { + const { runner, committer } = setup({ tipHeight: 1, archiveFailure: new Error("gcs down") }); + + const started = runner.start(); + started.catch(() => undefined); + await vi.runAllTimersAsync(); + + await expect(started).rejects.toThrow("gcs down"); + expect(committer.commit).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + }); + + describe("when the archive is disabled", () => { + it("syncs without touching the archive and logs the disabled state", async () => { + const { runner, archive, committer } = setup({ tipHeight: 2, archiveEnabled: false }); + + await runner.start(); + + expect(committer.commit).toHaveBeenCalledTimes(2); + expect(archive.putStagedBlockIfAbsent).not.toHaveBeenCalled(); + expect(archive.logState).toHaveBeenCalledTimes(1); + }); + }); + + function setup(input: { tipHeight: number; archiveEnabled?: boolean; archiveFailure?: Error }) { + const archiveEnabled = input.archiveEnabled ?? true; + const config = envSchema.parse({ + POSTGRES_DB_URI: "postgres://unit:unit@localhost:5432/unit", + SYNC_START_HEIGHT: "1", + ARCHIVE_BUCKET: archiveEnabled ? "raw-blocks" : "" + }); + + const dbFake = { + select: () => ({ + from: () => ({ + where: () => Promise.resolve([]) + }) + }) + }; + + const pool = mock(); + pool.getTipHeight.mockResolvedValue(input.tipHeight); + pool.getBlock.mockImplementation(async height => ({ block: { header: { height: String(height) } } }) as RpcBlockResult); + pool.getBlockResults.mockImplementation(async height => ({ height: String(height), txs_results: null })); + + const decoder = mock(); + decoder.decode.mockImplementation(block => buildDecodedBlock(parseInt(block.block.header.height))); + + const archive = mock(); + archive.isEnabled.mockReturnValue(archiveEnabled); + if (input.archiveFailure) { + archive.putStagedBlockIfAbsent.mockRejectedValue(input.archiveFailure); + } else { + archive.putStagedBlockIfAbsent.mockResolvedValue(undefined); + } + + const committer = mock(); + const logger = mock(); + const runner = new SyncRunnerService(dbFake as unknown as ChainDatabase, pool, decoder, committer, archive, config, logger); + committer.commit.mockImplementation(async decoded => { + if (decoded.height >= input.tipHeight) { + await runner.dispose(); + } + }); + + return { runner, archive, committer, logger, pool }; + } + + function buildDecodedBlock(height: number): DecodedBlock { + return { + height, + datetime: new Date("2026-08-12T00:00:00Z"), + hash: Buffer.from(`hash-${height}`), + parentHash: height > 1 ? Buffer.from(`hash-${height - 1}`) : null, + proposerAddress: "PROPOSER", + transactions: [] + }; + } +}); diff --git a/apps/chain-indexer/src/pipeline/sync-runner.service.ts b/apps/chain-indexer/src/pipeline/sync-runner.service.ts index 294eea08a5..35518c13b3 100644 --- a/apps/chain-indexer/src/pipeline/sync-runner.service.ts +++ b/apps/chain-indexer/src/pipeline/sync-runner.service.ts @@ -2,6 +2,8 @@ import { eq } from "drizzle-orm"; import { setTimeout as delay } from "node:timers/promises"; import { inject, singleton } from "tsyringe"; +import { fetchRawBlock } from "@src/archive/archive-layout"; +import { BlockArchiveService } from "@src/archive/block-archive.service"; import type { EnvConfig } from "@src/config/env.config"; import { Blocks, IndexerState } from "@src/db/schema"; import { BlockCommitterService, SYNC_STREAM } from "@src/pipeline/block-committer.service"; @@ -23,6 +25,7 @@ export class SyncRunnerService { readonly #pool: RpcClientPool; readonly #decoder: BlockDecoderService; readonly #committer: BlockCommitterService; + readonly #archive: BlockArchiveService; readonly #config: EnvConfig; readonly #logger: LoggerService; @@ -34,6 +37,7 @@ export class SyncRunnerService { @inject(RpcClientPool) pool: RpcClientPool, @inject(BlockDecoderService) decoder: BlockDecoderService, @inject(BlockCommitterService) committer: BlockCommitterService, + @inject(BlockArchiveService) archive: BlockArchiveService, @inject(APP_CONFIG) config: EnvConfig, @inject(LoggerService) logger: LoggerService ) { @@ -41,6 +45,7 @@ export class SyncRunnerService { this.#pool = pool; this.#decoder = decoder; this.#committer = committer; + this.#archive = archive; this.#config = config; this.#logger = logger; this.#logger.setContext("SYNC"); @@ -65,6 +70,7 @@ export class SyncRunnerService { async #run(): Promise { let nextHeight = await this.#resolveStartHeight(); this.#logger.info({ event: "SYNC_STARTED", network: this.#config.NETWORK, nextHeight }); + this.#archive.logState(); while (!this.#stopped) { const tipHeight = await this.#retryTransient(() => this.#pool.getTipHeight(), { event: "SYNC_TIP_FETCH_RETRY" }); @@ -86,9 +92,15 @@ export class SyncRunnerService { return await retryTransient(operation, { isStopped: () => this.#stopped, logger: this.#logger, logContext }); } + /** The raw payloads are archived before decode and commit, so no block is ever committed without being archived and raw blocks survive even decoder bugs. */ async #syncBlock(height: number): Promise { - const [block, blockResults] = await Promise.all([this.#pool.getBlock(height), this.#pool.getBlockResults(height)]); - const decoded = this.#decoder.decode(block, blockResults); + const record = await fetchRawBlock(this.#pool, height); + + if (this.#archive.isEnabled()) { + await this.#archive.putStagedBlockIfAbsent(record); + } + + const decoded = this.#decoder.decode(record.block, record.block_results); this.#verifyContinuity(decoded); await this.#committer.commit(decoded); diff --git a/apps/chain-indexer/src/providers/archive.provider.spec.ts b/apps/chain-indexer/src/providers/archive.provider.spec.ts new file mode 100644 index 0000000000..ea022ad279 --- /dev/null +++ b/apps/chain-indexer/src/providers/archive.provider.spec.ts @@ -0,0 +1,38 @@ +import { Storage } from "@google-cloud/storage"; +import { container } from "tsyringe"; +import { describe, expect, it } from "vitest"; + +import { ARCHIVE_STORAGE } from "@src/providers/archive.provider"; +import { RAW_APP_CONFIG } from "@src/providers/raw-app-config.provider"; + +describe("ARCHIVE_STORAGE", () => { + it("resolves a Storage client when ARCHIVE_BUCKET is set", () => { + const child = setup({ ARCHIVE_BUCKET: "raw-blocks" }); + + expect(child.resolve(ARCHIVE_STORAGE)).toBeInstanceOf(Storage); + }); + + it("resolves null when ARCHIVE_BUCKET is unset", () => { + const child = setup({}); + + expect(child.resolve(ARCHIVE_STORAGE)).toBeNull(); + }); + + it("caches the client per container", () => { + const child = setup({ ARCHIVE_BUCKET: "raw-blocks" }); + + expect(child.resolve(ARCHIVE_STORAGE)).toBe(child.resolve(ARCHIVE_STORAGE)); + }); + + it("points the client at ARCHIVE_STORAGE_API_ENDPOINT when set", () => { + const child = setup({ ARCHIVE_BUCKET: "raw-blocks", ARCHIVE_STORAGE_API_ENDPOINT: "http://localhost:4443" }); + + expect((child.resolve(ARCHIVE_STORAGE) as Storage).apiEndpoint).toBe("http://localhost:4443"); + }); + + function setup(env: Record) { + const child = container.createChildContainer(); + child.register(RAW_APP_CONFIG, { useValue: { POSTGRES_DB_URI: "postgres://unit:unit@localhost:5432/unit", ...env } }); + return child; + } +}); diff --git a/apps/chain-indexer/src/providers/archive.provider.ts b/apps/chain-indexer/src/providers/archive.provider.ts new file mode 100644 index 0000000000..8d3b37eed0 --- /dev/null +++ b/apps/chain-indexer/src/providers/archive.provider.ts @@ -0,0 +1,41 @@ +import { Storage } from "@google-cloud/storage"; +import type { DependencyContainer, InjectionToken } from "tsyringe"; +import { container, instancePerContainerCachingFactory } from "tsyringe"; + +import { APP_CONFIG } from "@src/providers/app-config.provider"; + +/** + * The narrow slice of the GCS SDK the archive uses, so tests can hand-roll an in-memory + * implementation without mocking the SDK's fluent surface. + */ +export interface ArchiveObjectStore { + bucket(name: string): { + file(key: string): { + save(data: Buffer, options: { resumable: boolean; contentType: string; preconditionOpts: { ifGenerationMatch: number } }): Promise; + download(): Promise<[Buffer]>; + delete(options?: { ignoreNotFound?: boolean }): Promise; + }; + }; +} + +/** + * autoRetry is off because the pipeline's own retry wrappers are the single retry authority; + * SDK-level retries would multiply attempts and stretch the effective timeout. + */ +const createArchiveStorage = (c: DependencyContainer): ArchiveObjectStore | null => { + const config = c.resolve(APP_CONFIG); + if (!config.ARCHIVE_BUCKET) { + return null; + } + return new Storage({ + retryOptions: { autoRetry: false }, + timeout: 30_000, + ...(config.ARCHIVE_STORAGE_API_ENDPOINT ? { apiEndpoint: config.ARCHIVE_STORAGE_API_ENDPOINT } : {}) + }); +}; + +export const ARCHIVE_STORAGE: InjectionToken = Symbol("ARCHIVE_STORAGE"); + +container.register(ARCHIVE_STORAGE, { + useFactory: instancePerContainerCachingFactory(createArchiveStorage) +}); diff --git a/apps/chain-indexer/src/providers/index.ts b/apps/chain-indexer/src/providers/index.ts index 31a971f1cb..0ba3aa7d2e 100644 --- a/apps/chain-indexer/src/providers/index.ts +++ b/apps/chain-indexer/src/providers/index.ts @@ -3,3 +3,4 @@ export * from "./app-config.provider"; export * from "./logging.provider"; export * from "./type-registry.provider"; export * from "./db.provider"; +export * from "./archive.provider"; diff --git a/apps/chain-indexer/src/rpc/rpc-types.ts b/apps/chain-indexer/src/rpc/rpc-types.ts index fec8502dbd..7b3da76653 100644 --- a/apps/chain-indexer/src/rpc/rpc-types.ts +++ b/apps/chain-indexer/src/rpc/rpc-types.ts @@ -1,4 +1,7 @@ export interface RpcStatusResult { + node_info: { + network: string; + }; sync_info: { latest_block_height: string; }; diff --git a/apps/chain-indexer/test/fakes/build-raw-block-record.ts b/apps/chain-indexer/test/fakes/build-raw-block-record.ts new file mode 100644 index 0000000000..9ccbafb8cf --- /dev/null +++ b/apps/chain-indexer/test/fakes/build-raw-block-record.ts @@ -0,0 +1,15 @@ +import type { RawBlockRecord } from "@src/archive/archive-layout"; + +export function buildRawBlockRecord(height: number, extra?: Record): RawBlockRecord { + return { + height, + block: { + block_id: { hash: `HASH-${height}` }, + block: { + header: { height: String(height), time: "2026-08-12T00:00:00Z", proposer_address: "PROP" }, + data: { txs: extra ? [JSON.stringify(extra)] : [] } + } + }, + block_results: { height: String(height), txs_results: null } + }; +} diff --git a/apps/chain-indexer/test/fakes/in-memory-object-store.ts b/apps/chain-indexer/test/fakes/in-memory-object-store.ts new file mode 100644 index 0000000000..244bff6aed --- /dev/null +++ b/apps/chain-indexer/test/fakes/in-memory-object-store.ts @@ -0,0 +1,57 @@ +import type { ArchiveObjectStore } from "@src/providers/archive.provider"; + +/** + * Stateful stand-in for the GCS SDK slice the archive uses, with real generation semantics: + * save with ifGenerationMatch 0 throws code 412 when the key already exists, download throws + * code 404 when absent, delete honors ignoreNotFound. Errors carry numeric `code` like the + * SDK's ApiError so the service's duck-typed detection works unchanged. + */ +export class InMemoryObjectStore implements ArchiveObjectStore { + readonly objects = new Map(); + failNextSaveWith: Error | null = null; + failNextDownloadWith: Error | null = null; + failNextDeleteWith: Error | null = null; + + bucket(name: string): ReturnType { + return { + file: (key: string) => ({ + save: async (data: Buffer, options: { resumable: boolean; contentType: string; preconditionOpts: { ifGenerationMatch: number } }) => { + this.#throwInjected("failNextSaveWith"); + const objectKey = `${name}/${key}`; + if (options.preconditionOpts.ifGenerationMatch === 0 && this.objects.has(objectKey)) { + throw httpError(412, `object ${objectKey} already exists`); + } + this.objects.set(objectKey, Buffer.from(data)); + }, + download: async (): Promise<[Buffer]> => { + this.#throwInjected("failNextDownloadWith"); + const data = this.objects.get(`${name}/${key}`); + if (!data) { + throw httpError(404, `object ${name}/${key} not found`); + } + return [data]; + }, + delete: async (options?: { ignoreNotFound?: boolean }) => { + this.#throwInjected("failNextDeleteWith"); + const objectKey = `${name}/${key}`; + if (!this.objects.has(objectKey) && !options?.ignoreNotFound) { + throw httpError(404, `object ${objectKey} not found`); + } + this.objects.delete(objectKey); + } + }) + }; + } + + #throwInjected(knob: "failNextSaveWith" | "failNextDownloadWith" | "failNextDeleteWith"): void { + const error = this[knob]; + if (error) { + this[knob] = null; + throw error; + } + } +} + +export function httpError(code: number, message: string): Error { + return Object.assign(new Error(message), { code }); +} diff --git a/apps/provider-inventory/CONTEXT.md b/apps/provider-inventory/CONTEXT.md index 4f5844c878..6b41895557 100644 --- a/apps/provider-inventory/CONTEXT.md +++ b/apps/provider-inventory/CONTEXT.md @@ -15,7 +15,7 @@ Live cluster state for a provider — nodes, GPUs, storage pools, and their allo _Avoid_: provider snapshot, current snapshot **provider snapshot**: -The legacy append-only history written by the 15-min poll in `apps/indexer`. Powers daily uptime graphs, dashboards, and the GPU repo. **Not** read by bid screening. +The legacy append-only history written by the 15-min poll in `apps/indexer`. Powers daily uptime graphs, dashboards, and the GPU repo. **Not** read by bid screening. Ownership of this off-chain history — snapshots, uptime, IP geolocation, and the GPU breakdown derived from inventory — is moving into `apps/provider-inventory` (it is deliberately kept out of the new `apps/chain-indexer`, which owns chain-derived data only); update this entry once that takeover lands. _Avoid_: provider inventory **node**: diff --git a/package-lock.json b/package-lock.json index 7443a4815f..7b01409ede 100644 --- a/package-lock.json +++ b/package-lock.json @@ -544,6 +544,7 @@ "@cosmjs/encoding": "~0.38.0", "@cosmjs/proto-signing": "~0.38.0", "@cosmjs/stargate": "~0.38.0", + "@google-cloud/storage": "^7.21.0", "@hono/node-server": "1.13.7", "@hono/otel": "~0.4.0", "@hono/zod-openapi": "0.18.4", @@ -560,7 +561,7 @@ "devDependencies": { "@akashnetwork/dev-config": "*", "@types/lodash": "^4.17.0", - "@types/node": "^22.13.11", + "@types/node": "^22.15.0", "@typescript-eslint/eslint-plugin": "^8.64.0", "@vitest/coverage-v8": "^4.1.5", "cosmjs-types": "~0.11.0", @@ -10609,6 +10610,74 @@ "@nestjs/core": "^10.x || ^11.0.0" } }, + "node_modules/@google-cloud/paginator": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.2.tgz", + "integrity": "sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==", + "license": "Apache-2.0", + "dependencies": { + "arrify": "^2.0.0", + "extend": "^3.0.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/projectify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", + "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/promisify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", + "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@google-cloud/storage": { + "version": "7.21.0", + "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-7.21.0.tgz", + "integrity": "sha512-l+IFTkd+6Y5LoAuXyYCKNAKtw/Ci+rAMqgdTB1jv4iZiLhw0rtq+0qjIRbBizXkNzEFmXiXUW0H7sZQQvk1ffA==", + "license": "Apache-2.0", + "dependencies": { + "@google-cloud/paginator": "^5.0.0", + "@google-cloud/projectify": "^4.0.0", + "@google-cloud/promisify": "<4.1.0", + "abort-controller": "^3.0.0", + "async-retry": "^1.3.3", + "duplexify": "^4.1.3", + "fast-xml-parser": "^5.3.4", + "gaxios": "^6.0.2", + "google-auth-library": "^9.6.3", + "html-entities": "^2.5.2", + "mime": "^3.0.0", + "p-limit": "^3.0.1", + "retry-request": "^7.0.0", + "teeny-request": "^9.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@google-cloud/storage/node_modules/mime": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", + "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/@grpc/grpc-js": { "version": "1.14.3", "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.3.tgz", @@ -13597,6 +13666,18 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/@nodable/entities": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.0.0.tgz", + "integrity": "sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "license": "MIT", @@ -21062,7 +21143,6 @@ }, "node_modules/@tootallnate/once": { "version": "2.0.0", - "dev": true, "license": "MIT", "engines": { "node": ">= 10" @@ -21169,6 +21249,12 @@ "@types/node": "*" } }, + "node_modules/@types/caseless": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", + "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==", + "license": "MIT" + }, "node_modules/@types/chai": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", @@ -21597,6 +21683,47 @@ "@types/d3-path": "^1" } }, + "node_modules/@types/request": { + "version": "2.48.13", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.13.tgz", + "integrity": "sha512-FGJ6udDNUCjd19pp0Q3iTiDkwhYup7J8hpMW9c4k53NrccQFFWKRho6hvtPPEhnXWKvukfwAlB6DbDz4yhH5Gg==", + "license": "MIT", + "dependencies": { + "@types/caseless": "*", + "@types/node": "*", + "@types/tough-cookie": "*", + "form-data": "^2.5.5" + } + }, + "node_modules/@types/request/node_modules/form-data": { + "version": "2.5.6", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.6.tgz", + "integrity": "sha512-Ogz/E85h9tlfJzpI6TuFpGcHZFhLrb9Gw8wq9v40CxSCPnv7ahKr6Xgtkn0KYCDQJ8DNn5VoMO8EXr9V5PadyA==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.4", + "mime-types": "^2.1.35", + "safe-buffer": "^5.2.1" + }, + "engines": { + "node": ">= 0.12" + } + }, + "node_modules/@types/request/node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/@types/resolve": { "version": "1.17.1", "license": "MIT", @@ -21702,7 +21829,6 @@ }, "node_modules/@types/tough-cookie": { "version": "4.0.5", - "dev": true, "license": "MIT" }, "node_modules/@types/trusted-types": { @@ -23522,7 +23648,6 @@ }, "node_modules/abort-controller": { "version": "3.0.0", - "dev": true, "license": "MIT", "dependencies": { "event-target-shim": "^5.0.0" @@ -23762,6 +23887,18 @@ "node": ">= 8" } }, + "node_modules/anynum": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz", + "integrity": "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/append-field": { "version": "1.0.0", "license": "MIT" @@ -23957,6 +24094,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/asap": { "version": "2.0.6", "dev": true, @@ -24020,6 +24166,15 @@ "node": ">= 0.4" } }, + "node_modules/async-retry": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", + "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", + "license": "MIT", + "dependencies": { + "retry": "0.13.1" + } + }, "node_modules/async-sema": { "version": "3.1.1", "license": "MIT" @@ -28455,7 +28610,6 @@ }, "node_modules/event-target-shim": { "version": "5.0.1", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -28680,6 +28834,45 @@ "version": "3.0.1", "license": "MIT" }, + "node_modules/fast-xml-builder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.0.tgz", + "integrity": "sha512-F74cZEdCvuw9P41GAC3rod4X04jjWGM1JPEv/GWSqFTWLsdyMSBMBMlm9Hk3GLBgLBbdBNY8yee0pQh2RBVESQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.6.2", + "xml-naming": "^0.3.0" + } + }, + "node_modules/fast-xml-parser": { + "version": "5.10.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.10.1.tgz", + "integrity": "sha512-IEMIf7298kXuZSRFoGfMYrl7is8LpavODgbNz1cwIudv7KwVFnuU+UsMporfq6PD6aXSlawZlARiA3UywCTfMw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "@nodable/entities": "^3.0.0", + "fast-xml-builder": "^1.2.0", + "is-unsafe": "^2.0.0", + "path-expression-matcher": "^1.6.2", + "strnum": "^2.4.1", + "xml-naming": "^0.3.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fastq": { "version": "1.17.1", "license": "ISC", @@ -29119,6 +29312,58 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/gaxios": { + "version": "6.7.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", + "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/gaxios/node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/gaxios/node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/gcp-metadata": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz", + "integrity": "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^6.1.1", + "google-logging-utils": "^0.0.2", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/geist": { "version": "1.3.1", "license": "SIL OPEN FONT LICENSE", @@ -29393,6 +29638,53 @@ "csstype": "^3.0.10" } }, + "node_modules/google-auth-library": { + "version": "9.15.1", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.15.1.tgz", + "integrity": "sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^6.1.1", + "gcp-metadata": "^6.1.0", + "gtoken": "^7.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/google-auth-library/node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/google-auth-library/node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/google-logging-utils": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-0.0.2.tgz", + "integrity": "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, "node_modules/gopd": { "version": "1.2.0", "license": "MIT", @@ -29407,6 +29699,40 @@ "version": "4.2.11", "license": "ISC" }, + "node_modules/gtoken": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", + "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", + "license": "MIT", + "dependencies": { + "gaxios": "^6.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/gtoken/node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/gtoken/node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, "node_modules/gzip-size": { "version": "6.0.0", "dev": true, @@ -29909,6 +30235,22 @@ "node": ">=12" } }, + "node_modules/html-entities": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.6.0.tgz", + "integrity": "sha512-kig+rMn/QOVRvr7c86gQ8lWXq+Hkv6CbAH1hLu+RG338StTpE8Z0b44SDVaqVu7HGKf27frdmUYEs9hTUX/cLQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/mdevils" + }, + { + "type": "patreon", + "url": "https://patreon.com/mdevils" + } + ], + "license": "MIT" + }, "node_modules/html-escaper": { "version": "2.0.2", "dev": true, @@ -30840,6 +31182,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-unsafe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.0.tgz", + "integrity": "sha512-2LdV822R+wmI86unXA93WCFpL6g+av8ynWk0nrHyJqGop5VoocYsSLFgN8jrfalT6iGeLNM4KXuVSsULP53kEA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/is-weakmap": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.2.tgz", @@ -31939,6 +32293,15 @@ "node": ">=6" } }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -36583,6 +36946,21 @@ "node": ">=8" } }, + "node_modules/path-expression-matcher": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.6.2.tgz", + "integrity": "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/path-is-absolute": { "version": "1.0.1", "license": "MIT", @@ -38699,10 +39077,33 @@ "node": ">=0.12" } }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, "node_modules/retry-as-promised": { "version": "7.0.4", "license": "MIT" }, + "node_modules/retry-request": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-7.0.2.tgz", + "integrity": "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==", + "license": "MIT", + "dependencies": { + "@types/request": "^2.48.8", + "extend": "^3.0.2", + "teeny-request": "^9.0.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/reusify": { "version": "1.0.4", "license": "MIT", @@ -39924,6 +40325,15 @@ "node": ">= 0.10.0" } }, + "node_modules/stream-events": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", + "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", + "license": "MIT", + "dependencies": { + "stubs": "^3.0.0" + } + }, "node_modules/stream-shift": { "version": "1.0.3", "license": "MIT" @@ -40214,6 +40624,27 @@ } } }, + "node_modules/strnum": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.4.1.tgz", + "integrity": "sha512-M9eUSMT2dCB2cTNPG7UYj6KuK7RJR2SN2+yCV/fTW3xzTCS6EaGZ5pSMgDIjB7r8zSfTGk+dvvn9rTjpVS9Mwg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "anynum": "^1.0.1" + } + }, + "node_modules/stubs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", + "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==", + "license": "MIT" + }, "node_modules/style-to-object": { "version": "0.4.4", "license": "MIT", @@ -40540,6 +40971,36 @@ "version": "4.0.0", "license": "ISC" }, + "node_modules/teeny-request": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-9.0.0.tgz", + "integrity": "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==", + "license": "Apache-2.0", + "dependencies": { + "http-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.9", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/teeny-request/node_modules/http-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", + "license": "MIT", + "dependencies": { + "@tootallnate/once": "2", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/temp-dir": { "version": "2.0.0", "license": "MIT", @@ -44154,6 +44615,21 @@ "node": ">=12" } }, + "node_modules/xml-naming": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz", + "integrity": "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/xmlchars": { "version": "2.2.0", "dev": true, From 6f4d27f0a1b29a09a17031e3126d2e30c3fe39cc Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Thu, 13 Aug 2026 01:13:25 +0530 Subject: [PATCH 05/14] feat(indexer): genesis import framework with account and balance seed (#3587) * feat(indexer): add genesis import framework with account and balance seed Balance tracking is only trustworthy from a network's genesis, so this seeds genesis state before the first block: accounts, per-denom balances (recorded as genesis-reason ledger entries), validators, and staking delegations, all in one transaction. It is gated behind GENESIS_IMPORT (default off). When enabled, the sync role runs a per-module seeder framework exactly once, using a genesis checkpoint in indexer_state so a restart is a no-op, and it rejects a fresh start that is not at the network's genesis height. Genesis is fetched over RPC /genesis_chunked and its chain_id is asserted against the node being indexed. This adds the accounts, account_balances, balance_changes, validators, and delegations tables (L-3); the ongoing per-block ledger lands in L-4. Refs CON-805 * refactor(indexer): share genesis validator and chunked-insert helpers Extract mapDescription/mapCommissionRates so the staking and gentx validator mappers stop duplicating the description and commission field mapping, and add an insertChunked helper the bank and staking seeders call instead of hand-rolling the chunked insert loop. * fix(indexer): warn when genesis import is enabled too late to seed The seed only runs on a fresh start, so turning GENESIS_IMPORT on after the indexer already has a sync checkpoint silently skips it and leaves the account and balance tables empty with no signal. Add a hasSeeded check on resume and log a warning so the operator knows genesis was never backfilled. --- apps/chain-indexer/README.md | 6 + .../drizzle/0001_long_mach_iv.sql | 55 ++ .../drizzle/meta/0001_snapshot.json | 695 ++++++++++++++++++ apps/chain-indexer/drizzle/meta/_journal.json | 7 + apps/chain-indexer/env/.env.sample | 1 + apps/chain-indexer/src/config/env.config.ts | 7 + .../chain-indexer/src/db/insert-chunk-size.ts | 2 + .../src/db/insert-chunked.spec.ts | 68 ++ apps/chain-indexer/src/db/insert-chunked.ts | 22 + apps/chain-indexer/src/db/schema.spec.ts | 46 ++ apps/chain-indexer/src/db/schema.ts | 108 ++- .../genesis/account-seeder.service.spec.ts | 53 ++ .../src/genesis/account-seeder.service.ts | 42 ++ .../src/genesis/bank-seeder.service.spec.ts | 60 ++ .../src/genesis/bank-seeder.service.ts | 44 ++ .../src/genesis/genesis-address.spec.ts | 22 + .../src/genesis/genesis-address.ts | 24 + .../genesis/genesis-import.service.spec.ts | 106 +++ .../src/genesis/genesis-import.service.ts | 106 +++ .../src/genesis/genesis-mid-chain-error.ts | 2 + .../src/genesis/genesis-schema.spec.ts | 117 +++ .../src/genesis/genesis-schema.ts | 280 +++++++ .../src/genesis/genesis-seed-context.ts | 13 + .../src/genesis/genesis-source.spec.ts | 62 ++ .../src/genesis/genesis-source.ts | 60 ++ .../genesis/staking-seeder.service.spec.ts | 61 ++ .../src/genesis/staking-seeder.service.ts | 44 ++ .../src/pipeline/block-committer.service.ts | 4 +- .../src/pipeline/sync-runner.service.spec.ts | 95 ++- .../src/pipeline/sync-runner.service.ts | 37 +- .../src/providers/db.provider.ts | 4 + .../src/rpc/rpc-client-pool.service.spec.ts | 10 + .../src/rpc/rpc-client-pool.service.ts | 6 +- apps/chain-indexer/src/rpc/rpc-types.ts | 7 + .../chain-indexer/test/fakes/build-tx-fake.ts | 39 + .../test/fakes/genesis-fixtures.ts | 112 +++ 36 files changed, 2410 insertions(+), 17 deletions(-) create mode 100644 apps/chain-indexer/drizzle/0001_long_mach_iv.sql create mode 100644 apps/chain-indexer/drizzle/meta/0001_snapshot.json create mode 100644 apps/chain-indexer/src/db/insert-chunk-size.ts create mode 100644 apps/chain-indexer/src/db/insert-chunked.spec.ts create mode 100644 apps/chain-indexer/src/db/insert-chunked.ts create mode 100644 apps/chain-indexer/src/db/schema.spec.ts create mode 100644 apps/chain-indexer/src/genesis/account-seeder.service.spec.ts create mode 100644 apps/chain-indexer/src/genesis/account-seeder.service.ts create mode 100644 apps/chain-indexer/src/genesis/bank-seeder.service.spec.ts create mode 100644 apps/chain-indexer/src/genesis/bank-seeder.service.ts create mode 100644 apps/chain-indexer/src/genesis/genesis-address.spec.ts create mode 100644 apps/chain-indexer/src/genesis/genesis-address.ts create mode 100644 apps/chain-indexer/src/genesis/genesis-import.service.spec.ts create mode 100644 apps/chain-indexer/src/genesis/genesis-import.service.ts create mode 100644 apps/chain-indexer/src/genesis/genesis-mid-chain-error.ts create mode 100644 apps/chain-indexer/src/genesis/genesis-schema.spec.ts create mode 100644 apps/chain-indexer/src/genesis/genesis-schema.ts create mode 100644 apps/chain-indexer/src/genesis/genesis-seed-context.ts create mode 100644 apps/chain-indexer/src/genesis/genesis-source.spec.ts create mode 100644 apps/chain-indexer/src/genesis/genesis-source.ts create mode 100644 apps/chain-indexer/src/genesis/staking-seeder.service.spec.ts create mode 100644 apps/chain-indexer/src/genesis/staking-seeder.service.ts create mode 100644 apps/chain-indexer/test/fakes/build-tx-fake.ts create mode 100644 apps/chain-indexer/test/fakes/genesis-fixtures.ts diff --git a/apps/chain-indexer/README.md b/apps/chain-indexer/README.md index fdfe0533f5..1e2d450838 100644 --- a/apps/chain-indexer/README.md +++ b/apps/chain-indexer/README.md @@ -40,6 +40,12 @@ curl localhost:3092/v1/status The checkpoint height should advance as blocks land in `cosmos.blocks`, `cosmos.transactions`, and `cosmos.messages`. +## Genesis import + +Set `GENESIS_IMPORT=true` to seed genesis state before the first block: accounts, per-denom balances (as `genesis`-reason ledger entries), validators, and staking delegations, all in one transaction. Because balance history is only trustworthy from the network's genesis, a fresh `sync` with the flag on must begin at the genesis height, and a fresh start anywhere else is rejected with a clear error. On sandbox that height is 1 (`SYNC_START_HEIGHT=1`); the height is read from the genesis file itself, so a chain continued from an export uses its continuation height. + +The import runs once. A `genesis` checkpoint in `indexer_state` makes a restart skip it, and the seed commits in a single transaction, so an interrupted run rolls back and retries cleanly. Genesis is fetched over RPC `/genesis_chunked` from the same nodes sync uses, and its `chain_id` must match the chain being indexed. Leave the flag unset (the default) and sync tails blocks, transactions, and messages from any height exactly as before. + ## Backfill The backfill role fills the database over an explicit, inclusive height range and exits when done, so it fits a one-off K8s Job: diff --git a/apps/chain-indexer/drizzle/0001_long_mach_iv.sql b/apps/chain-indexer/drizzle/0001_long_mach_iv.sql new file mode 100644 index 0000000000..7e98956454 --- /dev/null +++ b/apps/chain-indexer/drizzle/0001_long_mach_iv.sql @@ -0,0 +1,55 @@ +CREATE TYPE "cosmos"."balance_change_reason" AS ENUM('genesis', 'transfer', 'fee', 'reward', 'commission', 'slash', 'gov', 'ibc', 'escrow', 'bme', 'mint', 'burn');--> statement-breakpoint +CREATE TABLE "cosmos"."account_balances" ( + "account_id" integer NOT NULL, + "denom" text NOT NULL, + "amount" numeric(38, 0) NOT NULL, + CONSTRAINT "account_balances_account_id_denom_pk" PRIMARY KEY("account_id","denom") +); +--> statement-breakpoint +CREATE TABLE "cosmos"."accounts" ( + "id" serial PRIMARY KEY NOT NULL, + "address" text NOT NULL, + "account_number" bigint, + "account_type" text, + "is_module_account" boolean DEFAULT false NOT NULL +); +--> statement-breakpoint +CREATE TABLE "cosmos"."balance_changes" ( + "id" bigserial PRIMARY KEY NOT NULL, + "account_id" integer NOT NULL, + "denom" text NOT NULL, + "delta" numeric(38, 0) NOT NULL, + "balance_after" numeric(38, 0) NOT NULL, + "reason" "cosmos"."balance_change_reason" NOT NULL, + "height" bigint NOT NULL, + "counterparty_account_id" integer +); +--> statement-breakpoint +CREATE TABLE "cosmos"."delegations" ( + "delegator_account_id" integer NOT NULL, + "validator_operator_address" text NOT NULL, + "shares" numeric(38, 18) NOT NULL, + CONSTRAINT "delegations_delegator_account_id_validator_operator_address_pk" PRIMARY KEY("delegator_account_id","validator_operator_address") +); +--> statement-breakpoint +CREATE TABLE "cosmos"."validators" ( + "operator_address" text PRIMARY KEY NOT NULL, + "account_address" text, + "hex_address" text, + "moniker" text, + "identity" text, + "website" text, + "details" text, + "security_contact" text, + "commission_rate" numeric(20, 18), + "commission_max_rate" numeric(20, 18), + "commission_max_change_rate" numeric(20, 18), + "min_self_delegation" numeric(38, 0) +); +--> statement-breakpoint +ALTER TABLE "cosmos"."account_balances" ADD CONSTRAINT "account_balances_account_id_accounts_id_fk" FOREIGN KEY ("account_id") REFERENCES "cosmos"."accounts"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "cosmos"."balance_changes" ADD CONSTRAINT "balance_changes_account_id_accounts_id_fk" FOREIGN KEY ("account_id") REFERENCES "cosmos"."accounts"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "cosmos"."balance_changes" ADD CONSTRAINT "balance_changes_counterparty_account_id_accounts_id_fk" FOREIGN KEY ("counterparty_account_id") REFERENCES "cosmos"."accounts"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "cosmos"."delegations" ADD CONSTRAINT "delegations_delegator_account_id_accounts_id_fk" FOREIGN KEY ("delegator_account_id") REFERENCES "cosmos"."accounts"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +CREATE UNIQUE INDEX "accounts_address_idx" ON "cosmos"."accounts" USING btree ("address");--> statement-breakpoint +CREATE INDEX "balance_changes_account_denom_height_idx" ON "cosmos"."balance_changes" USING btree ("account_id","denom","height"); \ No newline at end of file diff --git a/apps/chain-indexer/drizzle/meta/0001_snapshot.json b/apps/chain-indexer/drizzle/meta/0001_snapshot.json new file mode 100644 index 0000000000..78e2d1fe28 --- /dev/null +++ b/apps/chain-indexer/drizzle/meta/0001_snapshot.json @@ -0,0 +1,695 @@ +{ + "id": "05402ce9-8c3f-4514-8150-e7911053df30", + "prevId": "332f29a4-e784-4238-9a9e-7139a844e7e2", + "version": "7", + "dialect": "postgresql", + "tables": { + "cosmos.account_balances": { + "name": "account_balances", + "schema": "cosmos", + "columns": { + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "amount": { + "name": "amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "account_balances_account_id_accounts_id_fk": { + "name": "account_balances_account_id_accounts_id_fk", + "tableFrom": "account_balances", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_balances_account_id_denom_pk": { + "name": "account_balances_account_id_denom_pk", + "columns": [ + "account_id", + "denom" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.accounts": { + "name": "accounts", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_number": { + "name": "account_number", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "account_type": { + "name": "account_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_module_account": { + "name": "is_module_account", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": { + "accounts_address_idx": { + "name": "accounts_address_idx", + "columns": [ + { + "expression": "address", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.balance_changes": { + "name": "balance_changes", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delta": { + "name": "delta", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "balance_after": { + "name": "balance_after", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "balance_change_reason", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "counterparty_account_id": { + "name": "counterparty_account_id", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "balance_changes_account_denom_height_idx": { + "name": "balance_changes_account_denom_height_idx", + "columns": [ + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "denom", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "balance_changes_account_id_accounts_id_fk": { + "name": "balance_changes_account_id_accounts_id_fk", + "tableFrom": "balance_changes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "balance_changes_counterparty_account_id_accounts_id_fk": { + "name": "balance_changes_counterparty_account_id_accounts_id_fk", + "tableFrom": "balance_changes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "counterparty_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.blocks": { + "name": "blocks", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "datetime": { + "name": "datetime", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "parent_hash": { + "name": "parent_hash", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "proposer_address": { + "name": "proposer_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tx_count": { + "name": "tx_count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.delegations": { + "name": "delegations", + "schema": "cosmos", + "columns": { + "delegator_account_id": { + "name": "delegator_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "validator_operator_address": { + "name": "validator_operator_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "shares": { + "name": "shares", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "delegations_delegator_account_id_accounts_id_fk": { + "name": "delegations_delegator_account_id_accounts_id_fk", + "tableFrom": "delegations", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "delegator_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "delegations_delegator_account_id_validator_operator_address_pk": { + "name": "delegations_delegator_account_id_validator_operator_address_pk", + "columns": [ + "delegator_account_id", + "validator_operator_address" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.indexer_state": { + "name": "indexer_state", + "schema": "", + "columns": { + "stream": { + "name": "stream", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "last_height": { + "name": "last_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.message_types": { + "name": "message_types", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "message_types_type_idx": { + "name": "message_types_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.messages": { + "name": "messages", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type_id": { + "name": "type_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "body": { + "name": "body", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "messages_type_id_idx": { + "name": "messages_type_id_idx", + "columns": [ + { + "expression": "type_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "messages_type_id_message_types_id_fk": { + "name": "messages_type_id_message_types_id_fk", + "tableFrom": "messages", + "tableTo": "message_types", + "schemaTo": "cosmos", + "columnsFrom": [ + "type_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messages_height_tx_index_index_pk": { + "name": "messages_height_tx_index_index_pk", + "columns": [ + "height", + "tx_index", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.transactions": { + "name": "transactions", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "code": { + "name": "code", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "gas_used": { + "name": "gas_used", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gas_wanted": { + "name": "gas_wanted", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "fee": { + "name": "fee", + "type": "jsonb", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "transactions_hash_idx": { + "name": "transactions_hash_idx", + "columns": [ + { + "expression": "hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "transactions_height_index_pk": { + "name": "transactions_height_index_pk", + "columns": [ + "height", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.validators": { + "name": "validators", + "schema": "cosmos", + "columns": { + "operator_address": { + "name": "operator_address", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_address": { + "name": "account_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "hex_address": { + "name": "hex_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "moniker": { + "name": "moniker", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "identity": { + "name": "identity", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "details": { + "name": "details", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "security_contact": { + "name": "security_contact", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commission_rate": { + "name": "commission_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "commission_max_rate": { + "name": "commission_max_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "commission_max_change_rate": { + "name": "commission_max_change_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "min_self_delegation": { + "name": "min_self_delegation", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "cosmos.balance_change_reason": { + "name": "balance_change_reason", + "schema": "cosmos", + "values": [ + "genesis", + "transfer", + "fee", + "reward", + "commission", + "slash", + "gov", + "ibc", + "escrow", + "bme", + "mint", + "burn" + ] + } + }, + "schemas": { + "cosmos": "cosmos" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/chain-indexer/drizzle/meta/_journal.json b/apps/chain-indexer/drizzle/meta/_journal.json index e539d4c4af..d80e14e60d 100644 --- a/apps/chain-indexer/drizzle/meta/_journal.json +++ b/apps/chain-indexer/drizzle/meta/_journal.json @@ -8,6 +8,13 @@ "when": 1786452233786, "tag": "0000_remarkable_scourge", "breakpoints": true + }, + { + "idx": 1, + "version": "7", + "when": 1786548202033, + "tag": "0001_long_mach_iv", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/chain-indexer/env/.env.sample b/apps/chain-indexer/env/.env.sample index 126216bcde..2c48dac39e 100644 --- a/apps/chain-indexer/env/.env.sample +++ b/apps/chain-indexer/env/.env.sample @@ -4,6 +4,7 @@ POSTGRES_DB_URI=postgres://user:password@localhost:5432/chain-indexer RPC_NODE_ENDPOINTS= SYNC_START_HEIGHT= SYNC_POLL_INTERVAL_MS=3000 +GENESIS_IMPORT=false BACKFILL_FROM_HEIGHT= BACKFILL_TO_HEIGHT= ARCHIVE_BUCKET= diff --git a/apps/chain-indexer/src/config/env.config.ts b/apps/chain-indexer/src/config/env.config.ts index 4915911c95..6f3c849257 100644 --- a/apps/chain-indexer/src/config/env.config.ts +++ b/apps/chain-indexer/src/config/env.config.ts @@ -14,6 +14,13 @@ const rawEnvSchema = z.object({ /** First height to sync when the database has no checkpoint yet. Defaults to the current chain tip. */ SYNC_START_HEIGHT: z.preprocess(emptyStringAsUndefined, z.number({ coerce: true }).int().positive().optional()), SYNC_POLL_INTERVAL_MS: z.number({ coerce: true }).int().positive().default(3_000), + /** + * Enables the one-time genesis import (accounts, balances, validators, delegations) before the first block. + * When on, a fresh sync must start at the network's genesis height or it is rejected as a mid-chain start. + * Off preserves plain block/tx/message tailing from any height. Enum-transform rather than z.coerce.boolean(), + * which treats the string "false" as true. + */ + GENESIS_IMPORT: z.preprocess(emptyStringAsUndefined, z.enum(["true", "false"]).default("false")).transform(value => value === "true"), /** First height of the backfill range (inclusive). Required when INDEXER_ROLE is "backfill". */ BACKFILL_FROM_HEIGHT: z.preprocess(emptyStringAsUndefined, z.number({ coerce: true }).int().positive().optional()), /** Last height of the backfill range (inclusive). Required when INDEXER_ROLE is "backfill". */ diff --git a/apps/chain-indexer/src/db/insert-chunk-size.ts b/apps/chain-indexer/src/db/insert-chunk-size.ts new file mode 100644 index 0000000000..6a31966b56 --- /dev/null +++ b/apps/chain-indexer/src/db/insert-chunk-size.ts @@ -0,0 +1,2 @@ +/** Keeps multi-row inserts well under postgres.js's ~65k bind-parameter limit. */ +export const INSERT_CHUNK_SIZE = 2_000; diff --git a/apps/chain-indexer/src/db/insert-chunked.spec.ts b/apps/chain-indexer/src/db/insert-chunked.spec.ts new file mode 100644 index 0000000000..a28bfc4c5a --- /dev/null +++ b/apps/chain-indexer/src/db/insert-chunked.spec.ts @@ -0,0 +1,68 @@ +import type { PgTable } from "drizzle-orm/pg-core"; +import { describe, expect, it } from "vitest"; + +import { INSERT_CHUNK_SIZE } from "@src/db/insert-chunk-size"; +import { insertChunked } from "@src/db/insert-chunked"; +import { AccountBalances } from "@src/db/schema"; +import type { ChainTransaction } from "@src/providers/db.provider"; + +describe(insertChunked.name, () => { + it("splits rows into chunks that stay within the insert limit and preserves their order", async () => { + const { tx, inserts } = setup(); + const rows = buildBalanceRows(INSERT_CHUNK_SIZE + 500); + + await insertChunked(tx, AccountBalances, rows); + + expect(inserts.map(insert => insert.rows.length)).toEqual([INSERT_CHUNK_SIZE, 500]); + expect(inserts.flatMap(insert => insert.rows)).toEqual(rows); + }); + + it("ignores conflicts on every chunk by default", async () => { + const { tx, inserts } = setup(); + + await insertChunked(tx, AccountBalances, buildBalanceRows(3)); + + expect(inserts.every(insert => insert.onConflict)).toBe(true); + }); + + it("writes every chunk without a conflict target when conflict handling is disabled", async () => { + const { tx, inserts } = setup(); + + await insertChunked(tx, AccountBalances, buildBalanceRows(3), { onConflictDoNothing: false }); + + expect(inserts.every(insert => insert.onConflict)).toBe(false); + }); + + it("issues no insert for an empty row set", async () => { + const { tx, inserts } = setup(); + + await insertChunked(tx, AccountBalances, []); + + expect(inserts).toEqual([]); + }); + + function buildBalanceRows(count: number): (typeof AccountBalances.$inferInsert)[] { + return Array.from({ length: count }, (_, index) => ({ accountId: index, denom: "uakt", amount: String(index) })); + } + + function setup() { + const inserts: { rows: Record[]; onConflict: boolean }[] = []; + const tx = { + insert(_table: PgTable) { + return { + values(rows: Record[]) { + const record = { rows, onConflict: false }; + inserts.push(record); + return Object.assign(Promise.resolve(), { + onConflictDoNothing: () => { + record.onConflict = true; + return Promise.resolve(); + } + }); + } + }; + } + }; + return { tx: tx as unknown as ChainTransaction, inserts }; + } +}); diff --git a/apps/chain-indexer/src/db/insert-chunked.ts b/apps/chain-indexer/src/db/insert-chunked.ts new file mode 100644 index 0000000000..0e1fde796f --- /dev/null +++ b/apps/chain-indexer/src/db/insert-chunked.ts @@ -0,0 +1,22 @@ +import type { PgTable } from "drizzle-orm/pg-core"; +import chunk from "lodash/chunk"; + +import { INSERT_CHUNK_SIZE } from "@src/db/insert-chunk-size"; +import type { ChainTransaction } from "@src/providers/db.provider"; + +/** + * Inserts `rows` in chunks that stay under postgres.js's bind-parameter limit. Conflicts are ignored by + * default so a re-seed is idempotent; pass `onConflictDoNothing: false` where an outer guard already + * enforces single-writing and every row must land (e.g. the genesis balance-change ledger). + */ +export async function insertChunked( + tx: ChainTransaction, + table: TTable, + rows: TTable["$inferInsert"][], + { onConflictDoNothing = true }: { onConflictDoNothing?: boolean } = {} +): Promise { + for (const rowChunk of chunk(rows, INSERT_CHUNK_SIZE)) { + const insert = tx.insert(table).values(rowChunk); + await (onConflictDoNothing ? insert.onConflictDoNothing() : insert); + } +} diff --git a/apps/chain-indexer/src/db/schema.spec.ts b/apps/chain-indexer/src/db/schema.spec.ts new file mode 100644 index 0000000000..80c7fe9dce --- /dev/null +++ b/apps/chain-indexer/src/db/schema.spec.ts @@ -0,0 +1,46 @@ +import { getTableConfig } from "drizzle-orm/pg-core"; +import { describe, expect, it } from "vitest"; + +import { AccountBalances, Accounts, BalanceChanges, Delegations, Validators } from "@src/db/schema"; + +describe("cosmos genesis schema", () => { + it("interns accounts under a unique address index", () => { + const config = getTableConfig(Accounts); + + expect(config.name).toBe("accounts"); + expect(config.columns.map(column => column.name)).toContain("is_module_account"); + expect(config.indexes).toHaveLength(1); + }); + + it("keys current balances by account and denom with an account foreign key", () => { + const config = getTableConfig(AccountBalances); + + expect(config.columns.map(column => column.name).sort()).toEqual(["account_id", "amount", "denom"]); + expect(config.primaryKeys).toHaveLength(1); + expect(config.foreignKeys).toHaveLength(1); + expect(config.foreignKeys[0].reference().foreignColumns[0].name).toBe("id"); + }); + + it("references accounts from the ledger for both the holder and the counterparty", () => { + const config = getTableConfig(BalanceChanges); + + expect(config.foreignKeys).toHaveLength(2); + expect(config.indexes).toHaveLength(1); + config.foreignKeys.forEach(foreignKey => expect(foreignKey.reference().foreignColumns[0].name).toBe("id")); + }); + + it("keys delegations by delegator and validator with a delegator foreign key", () => { + const config = getTableConfig(Delegations); + + expect(config.primaryKeys).toHaveLength(1); + expect(config.foreignKeys).toHaveLength(1); + expect(config.foreignKeys[0].reference().foreignColumns[0].name).toBe("id"); + }); + + it("keys validators by operator address", () => { + const config = getTableConfig(Validators); + + expect(config.name).toBe("validators"); + expect(config.columns.map(column => column.name)).toContain("operator_address"); + }); +}); diff --git a/apps/chain-indexer/src/db/schema.ts b/apps/chain-indexer/src/db/schema.ts index ea0591d979..93e9d6e184 100644 --- a/apps/chain-indexer/src/db/schema.ts +++ b/apps/chain-indexer/src/db/schema.ts @@ -1,4 +1,19 @@ -import { bigint, index, integer, jsonb, pgSchema, pgTable, primaryKey, serial, text, timestamp, uniqueIndex } from "drizzle-orm/pg-core"; +import { + bigint, + bigserial, + boolean, + index, + integer, + jsonb, + numeric, + pgSchema, + pgTable, + primaryKey, + serial, + text, + timestamp, + uniqueIndex +} from "drizzle-orm/pg-core"; import { bytea } from "@src/db/bytea"; @@ -60,3 +75,94 @@ export const IndexerState = pgTable("indexer_state", { lastHeight: bigint("last_height", { mode: "number" }).notNull(), updatedAt: timestamp("updated_at", { withTimezone: true }).notNull() }); + +/** + * Why an address change happened. Only `genesis` is written today (L-3); the ongoing per-block + * reasons (transfer/fee/staking/...) land with the balance ledger in L-4. New values are added via + * migration (Postgres allows ALTER TYPE ... ADD VALUE) rather than editing this list retroactively. + */ +export const balanceChangeReason = cosmosSchema.enum("balance_change_reason", [ + "genesis", + "transfer", + "fee", + "reward", + "commission", + "slash", + "gov", + "ibc", + "escrow", + "bme", + "mint", + "burn" +]); + +/** Addresses interned once and referenced by integer id, mirroring the message_types lookup. */ +export const Accounts = cosmosSchema.table( + "accounts", + { + id: serial("id").primaryKey(), + address: text("address").notNull(), + accountNumber: bigint("account_number", { mode: "number" }), + accountType: text("account_type"), + isModuleAccount: boolean("is_module_account").notNull().default(false) + }, + t => [uniqueIndex("accounts_address_idx").on(t.address)] +); + +/** Current per-denom balance for each account, updated in the same transaction as the ledger. */ +export const AccountBalances = cosmosSchema.table( + "account_balances", + { + accountId: integer("account_id") + .notNull() + .references(() => Accounts.id), + denom: text("denom").notNull(), + amount: numeric("amount", { precision: 38, scale: 0 }).notNull() + }, + t => [primaryKey({ columns: [t.accountId, t.denom] })] +); + +/** Append-only ledger of balance changes. `numeric(38,0)` never loses precision on u-denom amounts the way DOUBLE does. */ +export const BalanceChanges = cosmosSchema.table( + "balance_changes", + { + id: bigserial("id", { mode: "number" }).primaryKey(), + accountId: integer("account_id") + .notNull() + .references(() => Accounts.id), + denom: text("denom").notNull(), + delta: numeric("delta", { precision: 38, scale: 0 }).notNull(), + balanceAfter: numeric("balance_after", { precision: 38, scale: 0 }).notNull(), + reason: balanceChangeReason("reason").notNull(), + height: bigint("height", { mode: "number" }).notNull(), + counterpartyAccountId: integer("counterparty_account_id").references(() => Accounts.id) + }, + t => [index("balance_changes_account_denom_height_idx").on(t.accountId, t.denom, t.height)] +); + +export const Validators = cosmosSchema.table("validators", { + operatorAddress: text("operator_address").primaryKey(), + accountAddress: text("account_address"), + hexAddress: text("hex_address"), + moniker: text("moniker"), + identity: text("identity"), + website: text("website"), + details: text("details"), + securityContact: text("security_contact"), + commissionRate: numeric("commission_rate", { precision: 20, scale: 18 }), + commissionMaxRate: numeric("commission_max_rate", { precision: 20, scale: 18 }), + commissionMaxChangeRate: numeric("commission_max_change_rate", { precision: 20, scale: 18 }), + minSelfDelegation: numeric("min_self_delegation", { precision: 38, scale: 0 }) +}); + +export const Delegations = cosmosSchema.table( + "delegations", + { + delegatorAccountId: integer("delegator_account_id") + .notNull() + .references(() => Accounts.id), + validatorOperatorAddress: text("validator_operator_address").notNull(), + shares: numeric("shares", { precision: 38, scale: 18 }).notNull() + }, + t => [primaryKey({ columns: [t.delegatorAccountId, t.validatorOperatorAddress] })] +); diff --git a/apps/chain-indexer/src/genesis/account-seeder.service.spec.ts b/apps/chain-indexer/src/genesis/account-seeder.service.spec.ts new file mode 100644 index 0000000000..b4e3bced46 --- /dev/null +++ b/apps/chain-indexer/src/genesis/account-seeder.service.spec.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest"; + +import { Accounts } from "@src/db/schema"; +import { AccountSeeder } from "@src/genesis/account-seeder.service"; + +import { buildTxFake, rowsFor } from "@test/fakes/build-tx-fake"; +import { buildParsedGenesis } from "@test/fakes/genesis-fixtures"; + +describe(AccountSeeder.name, () => { + it("interns every genesis address once and returns the address→id map", async () => { + const { seeder, tx, inserts } = setup(); + + const idByAddress = await seeder.intern(tx, buildParsedGenesis()); + + expect([...idByAddress.keys()].sort()).toEqual(["akash1base", "akash1module", "akash1vesting"]); + expect(rowsFor(inserts, Accounts)).toHaveLength(3); + }); + + it("records account metadata including the module-account flag", async () => { + const { seeder, tx, inserts } = setup(); + + await seeder.intern(tx, buildParsedGenesis()); + + expect(rowsFor(inserts, Accounts)).toEqual([ + { address: "akash1base", accountNumber: 1, accountType: "base", isModuleAccount: false }, + { address: "akash1module", accountNumber: 2, accountType: "module", isModuleAccount: true }, + { address: "akash1vesting", accountNumber: 3, accountType: "vesting", isModuleAccount: false } + ]); + }); + + it("interns balance and delegator addresses that have no auth account entry", async () => { + const { seeder, tx, inserts } = setup(); + const genesis = { + ...buildParsedGenesis(), + accounts: [], + balances: [{ address: "akash1holder", coins: [{ denom: "uakt", amount: "1" }] }], + delegations: [{ delegatorAddress: "akash1delegator", validatorOperatorAddress: "akashvaloper1x", shares: "1" }] + }; + + const idByAddress = await seeder.intern(tx, genesis); + + expect([...idByAddress.keys()].sort()).toEqual(["akash1delegator", "akash1holder"]); + expect(rowsFor(inserts, Accounts)).toEqual([ + { address: "akash1holder", accountNumber: null, accountType: null, isModuleAccount: false }, + { address: "akash1delegator", accountNumber: null, accountType: null, isModuleAccount: false } + ]); + }); + + function setup() { + const { tx, inserts } = buildTxFake(); + return { seeder: new AccountSeeder(), tx, inserts }; + } +}); diff --git a/apps/chain-indexer/src/genesis/account-seeder.service.ts b/apps/chain-indexer/src/genesis/account-seeder.service.ts new file mode 100644 index 0000000000..27a61b26fb --- /dev/null +++ b/apps/chain-indexer/src/genesis/account-seeder.service.ts @@ -0,0 +1,42 @@ +import chunk from "lodash/chunk"; +import { singleton } from "tsyringe"; + +import { INSERT_CHUNK_SIZE } from "@src/db/insert-chunk-size"; +import { Accounts } from "@src/db/schema"; +import type { ParsedGenesis } from "@src/genesis/genesis-schema"; +import type { ChainTransaction } from "@src/providers/db.provider"; + +@singleton() +export class AccountSeeder { + /** + * Interns every address that appears in genesis — auth accounts, balance holders, and delegators — + * and returns the address→id map the other seeders reference. Genesis runs before block 1 against an + * empty accounts table, so `returning()` yields the full mapping without a follow-up select. + */ + async intern(tx: ChainTransaction, genesis: ParsedGenesis): Promise> { + const accountByAddress = new Map(genesis.accounts.map(account => [account.address, account])); + + const addresses = new Set(); + genesis.accounts.forEach(account => addresses.add(account.address)); + genesis.balances.forEach(balance => addresses.add(balance.address)); + genesis.delegations.forEach(delegation => addresses.add(delegation.delegatorAddress)); + + const rows: (typeof Accounts.$inferInsert)[] = [...addresses].map(address => { + const account = accountByAddress.get(address); + return { + address, + accountNumber: account?.accountNumber ?? null, + accountType: account?.accountType ?? null, + isModuleAccount: account?.isModuleAccount ?? false + }; + }); + + const idByAddress = new Map(); + for (const rowChunk of chunk(rows, INSERT_CHUNK_SIZE)) { + const inserted = await tx.insert(Accounts).values(rowChunk).onConflictDoNothing().returning({ id: Accounts.id, address: Accounts.address }); + inserted.forEach(row => idByAddress.set(row.address, row.id)); + } + + return idByAddress; + } +} diff --git a/apps/chain-indexer/src/genesis/bank-seeder.service.spec.ts b/apps/chain-indexer/src/genesis/bank-seeder.service.spec.ts new file mode 100644 index 0000000000..6774f23516 --- /dev/null +++ b/apps/chain-indexer/src/genesis/bank-seeder.service.spec.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; + +import { AccountBalances, BalanceChanges } from "@src/db/schema"; +import { BankSeeder } from "@src/genesis/bank-seeder.service"; + +import { buildTxFake, rowsFor } from "@test/fakes/build-tx-fake"; +import { buildParsedGenesis } from "@test/fakes/genesis-fixtures"; + +describe(BankSeeder.name, () => { + it("writes a current balance and a genesis ledger entry for each coin", async () => { + const { seeder, tx, inserts } = setup(); + + await seeder.seed(tx, buildParsedGenesis(), context()); + + expect(rowsFor(inserts, AccountBalances)).toEqual([ + { accountId: 1, denom: "uakt", amount: "10" }, + { accountId: 2, denom: "uakt", amount: "5" }, + { accountId: 3, denom: "uakt", amount: "20" } + ]); + expect(rowsFor(inserts, BalanceChanges)).toEqual([ + { accountId: 1, denom: "uakt", delta: "10", balanceAfter: "10", reason: "genesis", height: 1, counterpartyAccountId: null }, + { accountId: 2, denom: "uakt", delta: "5", balanceAfter: "5", reason: "genesis", height: 1, counterpartyAccountId: null }, + { accountId: 3, denom: "uakt", delta: "20", balanceAfter: "20", reason: "genesis", height: 1, counterpartyAccountId: null } + ]); + }); + + it("seeds current balances that total the genesis supply", async () => { + const { seeder, tx, inserts } = setup(); + const genesis = buildParsedGenesis(); + + await seeder.seed(tx, genesis, context()); + + const seededTotal = rowsFor(inserts, AccountBalances).reduce((sum, row) => sum + BigInt(row.amount as string), 0n); + const supplyTotal = genesis.supply.reduce((sum, coin) => sum + BigInt(coin.amount), 0n); + expect(seededTotal).toBe(supplyTotal); + }); + + it("throws when a balance address was not interned", async () => { + const { seeder, tx } = setup(); + const genesis = { ...buildParsedGenesis(), balances: [{ address: "akash1missing", coins: [{ denom: "uakt", amount: "1" }] }] }; + + await expect(seeder.seed(tx, genesis, context())).rejects.toThrow("No interned account id for balance address akash1missing"); + }); + + function context() { + return { + accountIdByAddress: new Map([ + ["akash1base", 1], + ["akash1module", 2], + ["akash1vesting", 3] + ]), + initialHeight: 1 + }; + } + + function setup() { + const { tx, inserts } = buildTxFake(); + return { seeder: new BankSeeder(), tx, inserts }; + } +}); diff --git a/apps/chain-indexer/src/genesis/bank-seeder.service.ts b/apps/chain-indexer/src/genesis/bank-seeder.service.ts new file mode 100644 index 0000000000..5b27d5f690 --- /dev/null +++ b/apps/chain-indexer/src/genesis/bank-seeder.service.ts @@ -0,0 +1,44 @@ +import { singleton } from "tsyringe"; + +import { insertChunked } from "@src/db/insert-chunked"; +import { AccountBalances, BalanceChanges } from "@src/db/schema"; +import type { ParsedGenesis } from "@src/genesis/genesis-schema"; +import type { GenesisModuleSeeder, GenesisSeedContext } from "@src/genesis/genesis-seed-context"; +import type { ChainTransaction } from "@src/providers/db.provider"; + +@singleton() +export class BankSeeder implements GenesisModuleSeeder { + /** + * Seeds every genesis balance as both a current-balance row and a `genesis`-reason ledger entry. + * Seeding all `bank.balances` (module and vesting accounts included) makes the current-balance total + * reconcile to `bank.supply` by construction. Idempotency comes from the import marker, so the ledger + * insert intentionally has no conflict target. + */ + async seed(tx: ChainTransaction, genesis: ParsedGenesis, context: GenesisSeedContext): Promise { + const balanceRows: (typeof AccountBalances.$inferInsert)[] = []; + const changeRows: (typeof BalanceChanges.$inferInsert)[] = []; + + for (const balance of genesis.balances) { + const accountId = context.accountIdByAddress.get(balance.address); + if (accountId === undefined) { + throw new Error(`No interned account id for balance address ${balance.address}`); + } + + for (const coin of balance.coins) { + balanceRows.push({ accountId, denom: coin.denom, amount: coin.amount }); + changeRows.push({ + accountId, + denom: coin.denom, + delta: coin.amount, + balanceAfter: coin.amount, + reason: "genesis", + height: context.initialHeight, + counterpartyAccountId: null + }); + } + } + + await insertChunked(tx, AccountBalances, balanceRows); + await insertChunked(tx, BalanceChanges, changeRows, { onConflictDoNothing: false }); + } +} diff --git a/apps/chain-indexer/src/genesis/genesis-address.spec.ts b/apps/chain-indexer/src/genesis/genesis-address.spec.ts new file mode 100644 index 0000000000..b718651786 --- /dev/null +++ b/apps/chain-indexer/src/genesis/genesis-address.spec.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from "vitest"; + +import { consensusHexAddress, operatorToAccountAddress } from "@src/genesis/genesis-address"; + +const ED25519_PUBKEY_TYPE = "/cosmos.crypto.ed25519.PubKey"; +const SANDBOX_VALIDATOR_PUBKEY = "1YM8H2iPYXxzSEQeFJQipwRnWV4sB2EKgujqdeTYLJs="; + +describe("consensusHexAddress", () => { + it("derives the uppercased hex consensus address from an ed25519 pubkey", () => { + expect(consensusHexAddress(ED25519_PUBKEY_TYPE, SANDBOX_VALIDATOR_PUBKEY)).toBe("31410FDD5FF7717918AB0D32645E12B6863B2576"); + }); + + it("throws for a non-ed25519 pubkey type", () => { + expect(() => consensusHexAddress("/cosmos.crypto.secp256k1.PubKey", SANDBOX_VALIDATOR_PUBKEY)).toThrow("Unsupported consensus pubkey type"); + }); +}); + +describe("operatorToAccountAddress", () => { + it("re-encodes a valoper address as its account address over the same bytes", () => { + expect(operatorToAccountAddress("akashvaloper1dq9wvqemmpvanmwsdttajsn4hmtx5zk7cgw7cz")).toBe("akash1dq9wvqemmpvanmwsdttajsn4hmtx5zk7j2qcgg"); + }); +}); diff --git a/apps/chain-indexer/src/genesis/genesis-address.ts b/apps/chain-indexer/src/genesis/genesis-address.ts new file mode 100644 index 0000000000..bca386e366 --- /dev/null +++ b/apps/chain-indexer/src/genesis/genesis-address.ts @@ -0,0 +1,24 @@ +import { fromBase64, fromBech32, toBech32, toHex } from "@cosmjs/encoding"; +import { createHash } from "node:crypto"; + +const ED25519_PUBKEY_TYPE = "/cosmos.crypto.ed25519.PubKey"; + +/** + * Consensus (hex) address of a validator: the first 20 bytes of SHA-256 over the ed25519 pubkey, + * uppercased, matching CometBFT and the legacy indexer. Consensus keys are always ed25519, so an + * unexpected type is a hard error rather than a silently wrong address. + */ +export function consensusHexAddress(pubkeyType: string, pubkeyBase64: string): string { + if (pubkeyType !== ED25519_PUBKEY_TYPE) { + throw new Error(`Unsupported consensus pubkey type ${pubkeyType}`); + } + + const digest = createHash("sha256").update(fromBase64(pubkeyBase64)).digest(); + return toHex(digest.subarray(0, 20)).toUpperCase(); +} + +/** Re-encodes an operator (`akashvaloper…`) address as its account (`akash…`) address; the underlying bytes are identical. */ +export function operatorToAccountAddress(operatorAddress: string): string { + const { prefix, data } = fromBech32(operatorAddress); + return toBech32(prefix.replace(/valoper$/, ""), data); +} diff --git a/apps/chain-indexer/src/genesis/genesis-import.service.spec.ts b/apps/chain-indexer/src/genesis/genesis-import.service.spec.ts new file mode 100644 index 0000000000..db156e08f2 --- /dev/null +++ b/apps/chain-indexer/src/genesis/genesis-import.service.spec.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import { IndexerState } from "@src/db/schema"; +import type { AccountSeeder } from "@src/genesis/account-seeder.service"; +import type { BankSeeder } from "@src/genesis/bank-seeder.service"; +import { GenesisImportService } from "@src/genesis/genesis-import.service"; +import { GenesisMidChainError } from "@src/genesis/genesis-mid-chain-error"; +import type { GenesisSource } from "@src/genesis/genesis-source"; +import type { StakingSeeder } from "@src/genesis/staking-seeder.service"; +import type { ChainDatabase } from "@src/providers/db.provider"; +import type { LoggerService } from "@src/providers/logging.provider"; + +import { buildParsedGenesis } from "@test/fakes/genesis-fixtures"; + +describe(GenesisImportService.name, () => { + it("rejects a fresh start that is not at the genesis height", async () => { + const { service, accountSeeder } = setup(); + + await expect(service.ensureSeeded(500)).rejects.toBeInstanceOf(GenesisMidChainError); + expect(accountSeeder.intern).not.toHaveBeenCalled(); + }); + + it("skips seeding when the genesis marker already exists", async () => { + const { service, accountSeeder, bankSeeder, stakingSeeder } = setup({ existingMarker: true }); + + await service.ensureSeeded(1); + + expect(accountSeeder.intern).not.toHaveBeenCalled(); + expect(bankSeeder.seed).not.toHaveBeenCalled(); + expect(stakingSeeder.seed).not.toHaveBeenCalled(); + }); + + it("seeds all modules in one transaction and claims the marker at the genesis height", async () => { + const { service, accountSeeder, bankSeeder, stakingSeeder, markerInserts } = setup(); + + await service.ensureSeeded(1); + + expect(accountSeeder.intern).toHaveBeenCalledTimes(1); + expect(bankSeeder.seed).toHaveBeenCalledTimes(1); + expect(stakingSeeder.seed).toHaveBeenCalledTimes(1); + expect(markerInserts).toEqual([expect.objectContaining({ stream: "genesis", lastHeight: 1 })]); + }); + + it("still seeds when the genesis has unmodeled account types", async () => { + const { service, source, accountSeeder } = setup(); + source.fetchGenesis.mockResolvedValue({ ...buildParsedGenesis(), unknownAccountTypes: ["/cosmos.auth.v1beta1.SomethingNew"] }); + + await service.ensureSeeded(1); + + expect(accountSeeder.intern).toHaveBeenCalledTimes(1); + }); + + it("does not seed when another writer claimed the marker first", async () => { + const { service, accountSeeder, bankSeeder } = setup({ claimReturnsEmpty: true }); + + await service.ensureSeeded(1); + + expect(accountSeeder.intern).not.toHaveBeenCalled(); + expect(bankSeeder.seed).not.toHaveBeenCalled(); + }); + + describe("hasSeeded", () => { + it("reports true when the genesis marker exists", async () => { + const { service } = setup({ existingMarker: true }); + + expect(await service.hasSeeded()).toBe(true); + }); + + it("reports false when the genesis marker is absent", async () => { + const { service } = setup(); + + expect(await service.hasSeeded()).toBe(false); + }); + }); + + function setup(input?: { existingMarker?: boolean; claimReturnsEmpty?: boolean }) { + const source = mock(); + source.fetchGenesis.mockResolvedValue(buildParsedGenesis()); + + const accountSeeder = mock(); + accountSeeder.intern.mockResolvedValue(new Map([["akash1base", 1]])); + const bankSeeder = mock(); + const stakingSeeder = mock(); + + const markerInserts: Record[] = []; + const txFake = { + insert: (table: unknown) => ({ + values: (row: Record) => { + if (table === IndexerState) { + markerInserts.push(row); + } + return { onConflictDoNothing: () => ({ returning: () => Promise.resolve(input?.claimReturnsEmpty ? [] : [row]) }) }; + } + }) + }; + + const dbFake = { + select: () => ({ from: () => ({ where: () => Promise.resolve(input?.existingMarker ? [{ stream: "genesis", lastHeight: 1 }] : []) }) }), + transaction: (callback: (tx: unknown) => Promise) => callback(txFake) + }; + + const service = new GenesisImportService(dbFake as unknown as ChainDatabase, source, accountSeeder, bankSeeder, stakingSeeder, mock()); + return { service, source, accountSeeder, bankSeeder, stakingSeeder, markerInserts }; + } +}); diff --git a/apps/chain-indexer/src/genesis/genesis-import.service.ts b/apps/chain-indexer/src/genesis/genesis-import.service.ts new file mode 100644 index 0000000000..dc98f93096 --- /dev/null +++ b/apps/chain-indexer/src/genesis/genesis-import.service.ts @@ -0,0 +1,106 @@ +import { eq } from "drizzle-orm"; +import { inject, singleton } from "tsyringe"; + +import { IndexerState } from "@src/db/schema"; +import { AccountSeeder } from "@src/genesis/account-seeder.service"; +import { BankSeeder } from "@src/genesis/bank-seeder.service"; +import { GenesisMidChainError } from "@src/genesis/genesis-mid-chain-error"; +import type { GenesisSource } from "@src/genesis/genesis-source"; +import { GENESIS_SOURCE } from "@src/genesis/genesis-source"; +import { StakingSeeder } from "@src/genesis/staking-seeder.service"; +import type { ChainDatabase } from "@src/providers/db.provider"; +import { CHAIN_DB } from "@src/providers/db.provider"; +import { LoggerService } from "@src/providers/logging.provider"; + +export const GENESIS_STREAM = "genesis"; + +@singleton() +export class GenesisImportService { + readonly #db: ChainDatabase; + readonly #source: GenesisSource; + readonly #accountSeeder: AccountSeeder; + readonly #bankSeeder: BankSeeder; + readonly #stakingSeeder: StakingSeeder; + readonly #logger: LoggerService; + + constructor( + @inject(CHAIN_DB) db: ChainDatabase, + @inject(GENESIS_SOURCE) source: GenesisSource, + @inject(AccountSeeder) accountSeeder: AccountSeeder, + @inject(BankSeeder) bankSeeder: BankSeeder, + @inject(StakingSeeder) stakingSeeder: StakingSeeder, + @inject(LoggerService) logger: LoggerService + ) { + this.#db = db; + this.#source = source; + this.#accountSeeder = accountSeeder; + this.#bankSeeder = bankSeeder; + this.#stakingSeeder = stakingSeeder; + this.#logger = logger; + this.#logger.setContext("GENESIS_IMPORT"); + } + + /** + * Seeds genesis state exactly once, before the first block. Rejects a fresh balance-tracking start + * whose start height is not the network's genesis height, so balances can never begin mid-chain. + * Safe to call on every fresh start: the marker row makes a repeat run a no-op, and the whole seed + * commits in one transaction so a crash mid-seed rolls back and retries cleanly. + */ + async ensureSeeded(startHeight: number): Promise { + const genesis = await this.#source.fetchGenesis(); + + if (startHeight !== genesis.initialHeight) { + throw new GenesisMidChainError( + `Balance tracking must start at genesis height ${genesis.initialHeight}, but the effective start height is ${startHeight}. Set SYNC_START_HEIGHT=${genesis.initialHeight} to index from genesis.` + ); + } + + const marker = await this.#findMarker(); + if (marker) { + this.#logger.info({ event: "GENESIS_ALREADY_SEEDED", height: marker.lastHeight }); + return; + } + + if (genesis.unknownAccountTypes.length > 0) { + this.#logger.warn({ event: "GENESIS_UNKNOWN_ACCOUNT_TYPES", types: genesis.unknownAccountTypes }); + } + + await this.#db.transaction(async tx => { + const claimed = await tx + .insert(IndexerState) + .values({ stream: GENESIS_STREAM, lastHeight: genesis.initialHeight, updatedAt: new Date() }) + .onConflictDoNothing() + .returning(); + + if (claimed.length === 0) { + this.#logger.info({ event: "GENESIS_SEED_SKIPPED_CONCURRENT" }); + return; + } + + const accountIdByAddress = await this.#accountSeeder.intern(tx, genesis); + const context = { accountIdByAddress, initialHeight: genesis.initialHeight }; + await this.#bankSeeder.seed(tx, genesis, context); + await this.#stakingSeeder.seed(tx, genesis, context); + + this.#logger.info({ + event: "GENESIS_SEEDED", + chainId: genesis.chainId, + initialHeight: genesis.initialHeight, + accounts: accountIdByAddress.size, + balances: genesis.balances.length, + validators: genesis.validators.length, + delegations: genesis.delegations.length + }); + }); + } + + /** Whether the one-time genesis seed has already run. Lets the sync runner detect a resume that turned the flag on too late to seed. */ + async hasSeeded(): Promise { + return (await this.#findMarker()) !== undefined; + } + + async #findMarker() { + const [marker] = await this.#db.select().from(IndexerState).where(eq(IndexerState.stream, GENESIS_STREAM)); + return marker; + } +} diff --git a/apps/chain-indexer/src/genesis/genesis-mid-chain-error.ts b/apps/chain-indexer/src/genesis/genesis-mid-chain-error.ts new file mode 100644 index 0000000000..c41047de78 --- /dev/null +++ b/apps/chain-indexer/src/genesis/genesis-mid-chain-error.ts @@ -0,0 +1,2 @@ +/** Raised when balance tracking would start past the network's genesis height; fatal by design so balances are never seeded from an incomplete history. */ +export class GenesisMidChainError extends Error {} diff --git a/apps/chain-indexer/src/genesis/genesis-schema.spec.ts b/apps/chain-indexer/src/genesis/genesis-schema.spec.ts new file mode 100644 index 0000000000..e38e498cc5 --- /dev/null +++ b/apps/chain-indexer/src/genesis/genesis-schema.spec.ts @@ -0,0 +1,117 @@ +import { describe, expect, it } from "vitest"; + +import { parseGenesis } from "@src/genesis/genesis-schema"; + +import { buildParsedGenesis, buildRawGenesis } from "@test/fakes/genesis-fixtures"; + +describe("parseGenesis", () => { + it("normalizes a genesis document into the flat parsed shape", () => { + expect(parseGenesis(buildRawGenesis())).toEqual(buildParsedGenesis()); + }); + + it("seeds balances that total the reported supply", () => { + const genesis = parseGenesis(buildRawGenesis()); + + const balanceTotal = genesis.balances.flatMap(balance => balance.coins).reduce((sum, coin) => sum + BigInt(coin.amount), 0n); + const supplyTotal = genesis.supply.reduce((sum, coin) => sum + BigInt(coin.amount), 0n); + + expect(balanceTotal).toBe(supplyTotal); + }); + + it("collects unmodeled account types instead of failing", () => { + const raw = { + chain_id: "sandbox-2", + initial_height: "1", + app_state: { + auth: { + accounts: [ + { "@type": "/cosmos.auth.v1beta1.BaseAccount", address: "akash1base", account_number: "1" }, + { "@type": "/cosmos.auth.v1beta1.SomethingNew", address: "akash1weird" } + ] + } + } + }; + + const genesis = parseGenesis(raw); + + expect(genesis.accounts).toEqual([{ address: "akash1base", accountNumber: 1, accountType: "base", isModuleAccount: false }]); + expect(genesis.unknownAccountTypes).toEqual(["/cosmos.auth.v1beta1.SomethingNew"]); + }); + + it("maps validators from the staking module export shape", () => { + const raw = { + chain_id: "akashnet-2", + initial_height: "9455001", + app_state: { + staking: { + validators: [ + { + operator_address: "akashvaloper1dq9wvqemmpvanmwsdttajsn4hmtx5zk7cgw7cz", + consensus_pubkey: { "@type": "/cosmos.crypto.ed25519.PubKey", key: "1YM8H2iPYXxzSEQeFJQipwRnWV4sB2EKgujqdeTYLJs=" }, + description: { moniker: "mainnet-val", identity: "id", website: "site", security_contact: "sc", details: "d" }, + commission: { commission_rates: { rate: "0.050000000000000000", max_rate: "0.200000000000000000", max_change_rate: "0.010000000000000000" } }, + min_self_delegation: "1000000" + } + ] + } + } + }; + + const genesis = parseGenesis(raw); + + expect(genesis.initialHeight).toBe(9455001); + expect(genesis.validators).toEqual([ + { + operatorAddress: "akashvaloper1dq9wvqemmpvanmwsdttajsn4hmtx5zk7cgw7cz", + accountAddress: "akash1dq9wvqemmpvanmwsdttajsn4hmtx5zk7j2qcgg", + hexAddress: "31410FDD5FF7717918AB0D32645E12B6863B2576", + moniker: "mainnet-val", + identity: "id", + website: "site", + details: "d", + securityContact: "sc", + commissionRate: "0.050000000000000000", + commissionMaxRate: "0.200000000000000000", + commissionMaxChangeRate: "0.010000000000000000", + minSelfDelegation: "1000000" + } + ]); + }); + + it("records a null account number when genesis omits it", () => { + const genesis = parseGenesis({ + chain_id: "sandbox-2", + initial_height: "1", + app_state: { auth: { accounts: [{ "@type": "/cosmos.auth.v1beta1.BaseAccount", address: "akash1noacctnum" }] } } + }); + + expect(genesis.accounts).toEqual([{ address: "akash1noacctnum", accountNumber: null, accountType: "base", isModuleAccount: false }]); + }); + + it("falls back to a null account address when the validator operator address is malformed", () => { + const genesis = parseGenesis({ + chain_id: "sandbox-2", + initial_height: "1", + app_state: { + staking: { validators: [{ operator_address: "invalid-operator", description: { moniker: "x" }, commission: {}, min_self_delegation: "1" }] } + } + }); + + expect(genesis.validators[0].accountAddress).toBeNull(); + expect(genesis.validators[0].hexAddress).toBeNull(); + }); + + it("defaults the initial height to 1 and tolerates missing modules", () => { + const genesis = parseGenesis({ chain_id: "sandbox-2", app_state: {} }); + + expect(genesis.initialHeight).toBe(1); + expect(genesis.accounts).toEqual([]); + expect(genesis.balances).toEqual([]); + expect(genesis.validators).toEqual([]); + expect(genesis.delegations).toEqual([]); + }); + + it("throws when a required top-level field is missing", () => { + expect(() => parseGenesis({ app_state: {} })).toThrow(); + }); +}); diff --git a/apps/chain-indexer/src/genesis/genesis-schema.ts b/apps/chain-indexer/src/genesis/genesis-schema.ts new file mode 100644 index 0000000000..0f04d3a685 --- /dev/null +++ b/apps/chain-indexer/src/genesis/genesis-schema.ts @@ -0,0 +1,280 @@ +import { z } from "zod"; + +import { consensusHexAddress, operatorToAccountAddress } from "@src/genesis/genesis-address"; + +const BASE_ACCOUNT_TYPE = "/cosmos.auth.v1beta1.BaseAccount"; +const MODULE_ACCOUNT_TYPE = "/cosmos.auth.v1beta1.ModuleAccount"; +const VESTING_TYPE_PREFIX = "/cosmos.vesting."; +const MSG_CREATE_VALIDATOR_TYPE = "/cosmos.staking.v1beta1.MsgCreateValidator"; + +export type AccountType = "base" | "module" | "vesting"; + +export interface ParsedCoin { + denom: string; + amount: string; +} + +export interface ParsedAccount { + address: string; + accountNumber: number | null; + accountType: AccountType | null; + isModuleAccount: boolean; +} + +export interface ParsedBalance { + address: string; + coins: ParsedCoin[]; +} + +export interface ParsedValidator { + operatorAddress: string; + accountAddress: string | null; + hexAddress: string | null; + moniker: string | null; + identity: string | null; + website: string | null; + details: string | null; + securityContact: string | null; + commissionRate: string | null; + commissionMaxRate: string | null; + commissionMaxChangeRate: string | null; + minSelfDelegation: string | null; +} + +export interface ParsedDelegation { + delegatorAddress: string; + validatorOperatorAddress: string; + shares: string; +} + +export interface ParsedGenesis { + chainId: string; + initialHeight: number; + genesisTime: string; + bondDenom: string | null; + accounts: ParsedAccount[]; + /** Account `@type`s we don't model, surfaced so the caller can log them without failing the import. */ + unknownAccountTypes: string[]; + balances: ParsedBalance[]; + supply: ParsedCoin[]; + validators: ParsedValidator[]; + delegations: ParsedDelegation[]; +} + +const coinSchema = z.object({ denom: z.string(), amount: z.string() }); + +const baseAccountInnerSchema = z.object({ address: z.string().optional(), account_number: z.string().optional() }).passthrough(); + +const rawAccountSchema = z + .object({ + "@type": z.string(), + address: z.string().optional(), + account_number: z.string().optional(), + base_account: baseAccountInnerSchema.optional(), + base_vesting_account: z.object({ base_account: baseAccountInnerSchema.optional() }).passthrough().optional() + }) + .passthrough(); + +const descriptionSchema = z + .object({ + moniker: z.string().optional(), + identity: z.string().optional(), + website: z.string().optional(), + security_contact: z.string().optional(), + details: z.string().optional() + }) + .partial() + .optional(); + +const pubkeySchema = z.object({ "@type": z.string(), key: z.string() }); + +const commissionRatesSchema = z.object({ rate: z.string().optional(), max_rate: z.string().optional(), max_change_rate: z.string().optional() }); + +const stakingValidatorSchema = z + .object({ + operator_address: z.string(), + consensus_pubkey: pubkeySchema.nullish(), + description: descriptionSchema, + commission: z.object({ commission_rates: commissionRatesSchema.optional() }).partial().optional(), + min_self_delegation: z.string().optional() + }) + .passthrough(); + +const createValidatorMsgSchema = z + .object({ + "@type": z.string(), + validator_address: z.string(), + delegator_address: z.string().optional(), + pubkey: pubkeySchema.nullish(), + description: descriptionSchema, + commission: commissionRatesSchema.optional(), + min_self_delegation: z.string().optional() + }) + .passthrough(); + +const delegationSchema = z.object({ delegator_address: z.string(), validator_address: z.string(), shares: z.string() }); + +const genesisSchema = z + .object({ + chain_id: z.string(), + initial_height: z.string().optional(), + genesis_time: z.string().optional(), + app_state: z + .object({ + auth: z + .object({ accounts: z.array(rawAccountSchema).optional() }) + .partial() + .optional(), + bank: z + .object({ + balances: z.array(z.object({ address: z.string(), coins: z.array(coinSchema) })).optional(), + supply: z.array(coinSchema).optional() + }) + .partial() + .optional(), + staking: z + .object({ + params: z.object({ bond_denom: z.string().optional() }).partial().optional(), + validators: z.array(stakingValidatorSchema).optional(), + delegations: z.array(delegationSchema).optional() + }) + .partial() + .optional(), + genutil: z + .object({ gen_txs: z.array(z.object({ body: z.object({ messages: z.array(z.record(z.unknown())) }).passthrough() }).passthrough()).optional() }) + .partial() + .optional() + }) + .passthrough() + }) + .passthrough(); + +type RawAccount = z.infer; +type RawStakingValidator = z.infer; +type RawCreateValidatorMsg = z.infer; + +/** Parses and validates the subset of a Cosmos-SDK genesis document the seeders need, normalizing snake_case + `@type` shapes into flat types. */ +export function parseGenesis(raw: unknown): ParsedGenesis { + const genesis = genesisSchema.parse(raw); + const appState = genesis.app_state; + + const unknownAccountTypes = new Set(); + const accounts: ParsedAccount[] = []; + for (const rawAccount of appState.auth?.accounts ?? []) { + const account = toParsedAccount(rawAccount); + if (account) { + accounts.push(account); + } else { + unknownAccountTypes.add(rawAccount["@type"]); + } + } + + return { + chainId: genesis.chain_id, + initialHeight: parseInt(genesis.initial_height ?? "1"), + genesisTime: genesis.genesis_time ?? "", + bondDenom: appState.staking?.params?.bond_denom ?? null, + accounts, + unknownAccountTypes: [...unknownAccountTypes], + balances: (appState.bank?.balances ?? []).map(balance => ({ address: balance.address, coins: balance.coins })), + supply: appState.bank?.supply ?? [], + validators: [...(appState.staking?.validators ?? []).map(toValidatorFromStaking), ...gentxValidators(appState.genutil?.gen_txs ?? [])], + delegations: (appState.staking?.delegations ?? []).map(delegation => ({ + delegatorAddress: delegation.delegator_address, + validatorOperatorAddress: delegation.validator_address, + shares: delegation.shares + })) + }; +} + +function toParsedAccount(raw: RawAccount): ParsedAccount | null { + const type = raw["@type"]; + + if (type === BASE_ACCOUNT_TYPE && raw.address) { + return { address: raw.address, accountNumber: toNumberOrNull(raw.account_number), accountType: "base", isModuleAccount: false }; + } + + if (type === MODULE_ACCOUNT_TYPE && raw.base_account?.address) { + return { address: raw.base_account.address, accountNumber: toNumberOrNull(raw.base_account.account_number), accountType: "module", isModuleAccount: true }; + } + + if (type.startsWith(VESTING_TYPE_PREFIX) && raw.base_vesting_account?.base_account?.address) { + return { + address: raw.base_vesting_account.base_account.address, + accountNumber: toNumberOrNull(raw.base_vesting_account.base_account.account_number), + accountType: "vesting", + isModuleAccount: false + }; + } + + return null; +} + +function toValidatorFromStaking(validator: RawStakingValidator): ParsedValidator { + return { + operatorAddress: validator.operator_address, + accountAddress: safeOperatorToAccountAddress(validator.operator_address), + hexAddress: validator.consensus_pubkey ? consensusHexAddress(validator.consensus_pubkey["@type"], validator.consensus_pubkey.key) : null, + ...mapDescription(validator.description), + ...mapCommissionRates(validator.commission?.commission_rates), + minSelfDelegation: validator.min_self_delegation ?? null + }; +} + +function gentxValidators(genTxs: { body: { messages: Record[] } }[]): ParsedValidator[] { + return genTxs + .flatMap(genTx => genTx.body.messages) + .filter(message => message["@type"] === MSG_CREATE_VALIDATOR_TYPE) + .map(message => toValidatorFromGentx(createValidatorMsgSchema.parse(message))); +} + +function toValidatorFromGentx(message: RawCreateValidatorMsg): ParsedValidator { + return { + operatorAddress: message.validator_address, + accountAddress: message.delegator_address ?? safeOperatorToAccountAddress(message.validator_address), + hexAddress: message.pubkey ? consensusHexAddress(message.pubkey["@type"], message.pubkey.key) : null, + ...mapDescription(message.description), + ...mapCommissionRates(message.commission), + minSelfDelegation: message.min_self_delegation ?? null + }; +} + +/** The staking export nests commission rates under `commission.commission_rates`; a gentx message puts them directly under `commission`. Both resolve to `commissionRatesSchema`, so callers pass whichever their shape exposes. */ +function mapCommissionRates( + rates: z.infer | undefined +): Pick { + return { + commissionRate: rates?.rate ?? null, + commissionMaxRate: rates?.max_rate ?? null, + commissionMaxChangeRate: rates?.max_change_rate ?? null + }; +} + +function mapDescription( + description: z.infer +): Pick { + return { + moniker: description?.moniker ?? null, + identity: description?.identity ?? null, + website: description?.website ?? null, + details: description?.details ?? null, + securityContact: description?.security_contact ?? null + }; +} + +function safeOperatorToAccountAddress(operatorAddress: string): string | null { + try { + return operatorToAccountAddress(operatorAddress); + } catch { + return null; + } +} + +function toNumberOrNull(value: string | undefined): number | null { + if (value === undefined) { + return null; + } + + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; +} diff --git a/apps/chain-indexer/src/genesis/genesis-seed-context.ts b/apps/chain-indexer/src/genesis/genesis-seed-context.ts new file mode 100644 index 0000000000..18371a3e65 --- /dev/null +++ b/apps/chain-indexer/src/genesis/genesis-seed-context.ts @@ -0,0 +1,13 @@ +import type { ParsedGenesis } from "@src/genesis/genesis-schema"; +import type { ChainTransaction } from "@src/providers/db.provider"; + +/** Shared context passed to each module seeder: the interned address→id map plus the genesis height. */ +export interface GenesisSeedContext { + accountIdByAddress: ReadonlyMap; + initialHeight: number; +} + +/** A per-module genesis seeder, matching the design's `ModuleDefinition.genesisSeeder`. Runs inside the shared import transaction. */ +export interface GenesisModuleSeeder { + seed(tx: ChainTransaction, genesis: ParsedGenesis, context: GenesisSeedContext): Promise; +} diff --git a/apps/chain-indexer/src/genesis/genesis-source.spec.ts b/apps/chain-indexer/src/genesis/genesis-source.spec.ts new file mode 100644 index 0000000000..7d5ec12982 --- /dev/null +++ b/apps/chain-indexer/src/genesis/genesis-source.spec.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import { RpcGenesisSource } from "@src/genesis/genesis-source"; +import type { LoggerService } from "@src/providers/logging.provider"; +import type { RpcClientPool } from "@src/rpc/rpc-client-pool.service"; + +import { buildParsedGenesis, buildRawGenesis } from "@test/fakes/genesis-fixtures"; + +describe(RpcGenesisSource.name, () => { + it("reassembles multiple genesis chunks into the parsed document", async () => { + const { source, pool } = setup({ chunkCount: 3 }); + + const genesis = await source.fetchGenesis(); + + expect(genesis).toEqual(buildParsedGenesis()); + expect(pool.getGenesisChunk).toHaveBeenCalledTimes(3); + expect(pool.getGenesisChunk).toHaveBeenNthCalledWith(1, 0); + expect(pool.getGenesisChunk).toHaveBeenNthCalledWith(3, 2); + }); + + it("fetches a single-chunk genesis", async () => { + const { source, pool } = setup({ chunkCount: 1 }); + + await source.fetchGenesis(); + + expect(pool.getGenesisChunk).toHaveBeenCalledTimes(1); + }); + + it("rejects when the genesis chain-id does not match the node", async () => { + const { source } = setup({ chunkCount: 1, nodeChainId: "othernet" }); + + await expect(source.fetchGenesis()).rejects.toThrow('Genesis chain_id "sandbox-2" does not match the RPC chain-id "othernet"'); + }); + + it("rejects an invalid chunk total", async () => { + const { source, pool } = setup({ chunkCount: 1 }); + pool.getGenesisChunk.mockResolvedValueOnce({ chunk: "0", total: "0", data: "" }); + + await expect(source.fetchGenesis()).rejects.toThrow("Invalid genesis chunk total"); + }); + + function setup(input: { chunkCount: number; nodeChainId?: string }) { + const encodedChunks = toBase64Chunks(JSON.stringify(buildRawGenesis()), input.chunkCount); + + const pool = mock(); + pool.getGenesisChunk.mockImplementation(async chunk => ({ chunk: String(chunk), total: String(encodedChunks.length), data: encodedChunks[chunk] })); + pool.getStatus.mockResolvedValue({ node_info: { network: input.nodeChainId ?? "sandbox-2" }, sync_info: { latest_block_height: "100" } }); + + const source = new RpcGenesisSource(pool, mock()); + return { source, pool }; + } + + function toBase64Chunks(json: string, count: number): string[] { + const size = Math.ceil(json.length / count); + const chunks: string[] = []; + for (let offset = 0; offset < json.length; offset += size) { + chunks.push(Buffer.from(json.slice(offset, offset + size)).toString("base64")); + } + return chunks; + } +}); diff --git a/apps/chain-indexer/src/genesis/genesis-source.ts b/apps/chain-indexer/src/genesis/genesis-source.ts new file mode 100644 index 0000000000..2a66f61881 --- /dev/null +++ b/apps/chain-indexer/src/genesis/genesis-source.ts @@ -0,0 +1,60 @@ +import { fromBase64 } from "@cosmjs/encoding"; +import type { InjectionToken } from "tsyringe"; +import { container, inject, singleton } from "tsyringe"; + +import type { ParsedGenesis } from "@src/genesis/genesis-schema"; +import { parseGenesis } from "@src/genesis/genesis-schema"; +import { LoggerService } from "@src/providers/logging.provider"; +import { RpcClientPool } from "@src/rpc/rpc-client-pool.service"; + +/** Seam over where the genesis document comes from. Swapping to a GitHub-mirror source (for a large mainnet genesis) is a one-line re-registration. */ +export interface GenesisSource { + fetchGenesis(): Promise; +} + +export const GENESIS_SOURCE: InjectionToken = Symbol("GENESIS_SOURCE"); + +@singleton() +export class RpcGenesisSource implements GenesisSource { + readonly #pool: RpcClientPool; + readonly #logger: LoggerService; + + constructor(@inject(RpcClientPool) pool: RpcClientPool, @inject(LoggerService) logger: LoggerService) { + this.#pool = pool; + this.#logger = logger; + this.#logger.setContext("GENESIS_SOURCE"); + } + + /** Fetches genesis from the same RPC pool the indexer syncs from and asserts its chain-id matches, so balances can only be seeded for the chain being indexed. */ + async fetchGenesis(): Promise { + const genesis = parseGenesis(await this.#fetchRawGenesis()); + const chainId = (await this.#pool.getStatus()).node_info.network; + + if (genesis.chainId !== chainId) { + throw new Error(`Genesis chain_id "${genesis.chainId}" does not match the RPC chain-id "${chainId}"`); + } + + return genesis; + } + + async #fetchRawGenesis(): Promise { + const first = await this.#pool.getGenesisChunk(0); + const total = Number(first.total); + + if (!Number.isInteger(total) || total < 1) { + throw new Error(`Invalid genesis chunk total: ${JSON.stringify(first.total)}`); + } + + const encodedChunks: string[] = [first.data]; + for (let chunk = 1; chunk < total; chunk++) { + encodedChunks.push((await this.#pool.getGenesisChunk(chunk)).data); + } + + const decoded = Buffer.concat(encodedChunks.map(encoded => Buffer.from(fromBase64(encoded)))); + this.#logger.info({ event: "GENESIS_FETCHED", chunks: total, bytes: decoded.byteLength }); + + return JSON.parse(decoded.toString("utf8")); + } +} + +container.register(GENESIS_SOURCE, { useToken: RpcGenesisSource }); diff --git a/apps/chain-indexer/src/genesis/staking-seeder.service.spec.ts b/apps/chain-indexer/src/genesis/staking-seeder.service.spec.ts new file mode 100644 index 0000000000..faf5efbf28 --- /dev/null +++ b/apps/chain-indexer/src/genesis/staking-seeder.service.spec.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; + +import { Delegations, Validators } from "@src/db/schema"; +import { StakingSeeder } from "@src/genesis/staking-seeder.service"; + +import { buildTxFake, rowsFor } from "@test/fakes/build-tx-fake"; +import { buildParsedGenesis } from "@test/fakes/genesis-fixtures"; + +describe(StakingSeeder.name, () => { + it("seeds validators from genesis", async () => { + const { seeder, tx, inserts } = setup(); + + await seeder.seed(tx, buildParsedGenesis(), context()); + + expect(rowsFor(inserts, Validators)).toEqual([ + expect.objectContaining({ + operatorAddress: "akashvaloper1dq9wvqemmpvanmwsdttajsn4hmtx5zk7cgw7cz", + accountAddress: "akash1dq9wvqemmpvanmwsdttajsn4hmtx5zk7j2qcgg", + hexAddress: "31410FDD5FF7717918AB0D32645E12B6863B2576", + moniker: "validator-01", + commissionRate: "0.100000000000000000", + minSelfDelegation: "1" + }) + ]); + }); + + it("resolves the delegator account id for each delegation", async () => { + const { seeder, tx, inserts } = setup(); + + await seeder.seed(tx, buildParsedGenesis(), context()); + + expect(rowsFor(inserts, Delegations)).toEqual([ + { delegatorAccountId: 1, validatorOperatorAddress: "akashvaloper1dq9wvqemmpvanmwsdttajsn4hmtx5zk7cgw7cz", shares: "1000000.000000000000000000" } + ]); + }); + + it("writes nothing when there are no validators or delegations", async () => { + const { seeder, tx, inserts } = setup(); + const genesis = { ...buildParsedGenesis(), validators: [], delegations: [] }; + + await seeder.seed(tx, genesis, context()); + + expect(inserts).toEqual([]); + }); + + it("throws when a delegator was not interned", async () => { + const { seeder, tx } = setup(); + const genesis = { ...buildParsedGenesis(), delegations: [{ delegatorAddress: "akash1missing", validatorOperatorAddress: "akashvaloper1x", shares: "1" }] }; + + await expect(seeder.seed(tx, genesis, context())).rejects.toThrow("No interned account id for delegator akash1missing"); + }); + + function context() { + return { accountIdByAddress: new Map([["akash1base", 1]]), initialHeight: 1 }; + } + + function setup() { + const { tx, inserts } = buildTxFake(); + return { seeder: new StakingSeeder(), tx, inserts }; + } +}); diff --git a/apps/chain-indexer/src/genesis/staking-seeder.service.ts b/apps/chain-indexer/src/genesis/staking-seeder.service.ts new file mode 100644 index 0000000000..1060e635e2 --- /dev/null +++ b/apps/chain-indexer/src/genesis/staking-seeder.service.ts @@ -0,0 +1,44 @@ +import { singleton } from "tsyringe"; + +import { insertChunked } from "@src/db/insert-chunked"; +import { Delegations, Validators } from "@src/db/schema"; +import type { ParsedGenesis } from "@src/genesis/genesis-schema"; +import type { GenesisModuleSeeder, GenesisSeedContext } from "@src/genesis/genesis-seed-context"; +import type { ChainTransaction } from "@src/providers/db.provider"; + +@singleton() +export class StakingSeeder implements GenesisModuleSeeder { + /** + * Seeds validators (from `staking.validators` and `genutil.gen_txs` create-validator messages) and + * explicit `staking.delegations`. Genesis gentx self-delegations are applied at InitChain rather than + * listed in `staking.delegations`, so they are intentionally not reconstructed here. + */ + async seed(tx: ChainTransaction, genesis: ParsedGenesis, context: GenesisSeedContext): Promise { + const validatorRows: (typeof Validators.$inferInsert)[] = genesis.validators.map(validator => ({ + operatorAddress: validator.operatorAddress, + accountAddress: validator.accountAddress, + hexAddress: validator.hexAddress, + moniker: validator.moniker, + identity: validator.identity, + website: validator.website, + details: validator.details, + securityContact: validator.securityContact, + commissionRate: validator.commissionRate, + commissionMaxRate: validator.commissionMaxRate, + commissionMaxChangeRate: validator.commissionMaxChangeRate, + minSelfDelegation: validator.minSelfDelegation + })); + + const delegationRows: (typeof Delegations.$inferInsert)[] = genesis.delegations.map(delegation => { + const delegatorAccountId = context.accountIdByAddress.get(delegation.delegatorAddress); + if (delegatorAccountId === undefined) { + throw new Error(`No interned account id for delegator ${delegation.delegatorAddress}`); + } + + return { delegatorAccountId, validatorOperatorAddress: delegation.validatorOperatorAddress, shares: delegation.shares }; + }); + + await insertChunked(tx, Validators, validatorRows); + await insertChunked(tx, Delegations, delegationRows); + } +} diff --git a/apps/chain-indexer/src/pipeline/block-committer.service.ts b/apps/chain-indexer/src/pipeline/block-committer.service.ts index 72c01f8f42..17935209f2 100644 --- a/apps/chain-indexer/src/pipeline/block-committer.service.ts +++ b/apps/chain-indexer/src/pipeline/block-committer.service.ts @@ -2,6 +2,7 @@ import { inArray, sql } from "drizzle-orm"; import chunk from "lodash/chunk"; import { inject, singleton } from "tsyringe"; +import { INSERT_CHUNK_SIZE } from "@src/db/insert-chunk-size"; import { Blocks, IndexerState, Messages, MessageTypes, Transactions } from "@src/db/schema"; import type { DecodedBlock } from "@src/pipeline/decoded-block"; import type { ChainDatabase } from "@src/providers/db.provider"; @@ -9,9 +10,6 @@ import { CHAIN_DB } from "@src/providers/db.provider"; export const SYNC_STREAM = "sync"; -/** Keeps multi-row inserts well under postgres.js's ~65k bind-parameter limit when batches span hundreds of blocks. */ -const INSERT_CHUNK_SIZE = 2_000; - @singleton() export class BlockCommitterService { readonly #db: ChainDatabase; diff --git a/apps/chain-indexer/src/pipeline/sync-runner.service.spec.ts b/apps/chain-indexer/src/pipeline/sync-runner.service.spec.ts index c8a446ce86..f998063442 100644 --- a/apps/chain-indexer/src/pipeline/sync-runner.service.spec.ts +++ b/apps/chain-indexer/src/pipeline/sync-runner.service.spec.ts @@ -3,6 +3,8 @@ import { mock } from "vitest-mock-extended"; import type { BlockArchiveService } from "@src/archive/block-archive.service"; import { envSchema } from "@src/config/env.config"; +import { Blocks, IndexerState } from "@src/db/schema"; +import type { GenesisImportService } from "@src/genesis/genesis-import.service"; import type { BlockCommitterService } from "@src/pipeline/block-committer.service"; import type { BlockDecoderService } from "@src/pipeline/block-decoder.service"; import type { DecodedBlock } from "@src/pipeline/decoded-block"; @@ -71,18 +73,96 @@ describe(SyncRunnerService.name, () => { }); }); - function setup(input: { tipHeight: number; archiveEnabled?: boolean; archiveFailure?: Error }) { + it("starts from the chain tip when no checkpoint or start height is configured", async () => { + const { runner, committer } = setup({ tipHeight: 4, omitStartHeight: true }); + + await runner.start(); + + expect(committer.commit).toHaveBeenCalledTimes(1); + expect(committer.commit).toHaveBeenCalledWith(expect.objectContaining({ height: 4 })); + }); + + describe("genesis import", () => { + it("runs the genesis import at the fresh start height when enabled", async () => { + const { runner, genesisImport } = setup({ tipHeight: 1, genesisImportEnabled: true }); + + await runner.start(); + + expect(genesisImport.ensureSeeded).toHaveBeenCalledWith(1); + }); + + it("does not run the genesis import when disabled", async () => { + const { runner, genesisImport } = setup({ tipHeight: 1 }); + + await runner.start(); + + expect(genesisImport.ensureSeeded).not.toHaveBeenCalled(); + }); + + it("does not run the genesis import when resuming from a checkpoint", async () => { + const { runner, genesisImport } = setup({ tipHeight: 2, genesisImportEnabled: true, checkpointHeight: 1 }); + + await runner.start(); + + expect(genesisImport.ensureSeeded).not.toHaveBeenCalled(); + }); + + it("halts before syncing when the genesis guard rejects a mid-chain start", async () => { + const { runner, genesisImport, committer } = setup({ tipHeight: 1, genesisImportEnabled: true }); + genesisImport.ensureSeeded.mockRejectedValue(new Error("mid-chain")); + + await expect(runner.start()).rejects.toThrow("mid-chain"); + expect(committer.commit).not.toHaveBeenCalled(); + }); + + it("warns when genesis import is enabled on resume but genesis was never seeded", async () => { + const { runner, genesisImport, logger } = setup({ tipHeight: 2, genesisImportEnabled: true, checkpointHeight: 1 }); + genesisImport.hasSeeded.mockResolvedValue(false); + + await runner.start(); + + expect(genesisImport.ensureSeeded).not.toHaveBeenCalled(); + expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ event: "GENESIS_IMPORT_SKIPPED_RESUMED_WITHOUT_MARKER" })); + }); + + it("does not warn when resuming an indexer that already seeded genesis", async () => { + const { runner, genesisImport, logger } = setup({ tipHeight: 2, genesisImportEnabled: true, checkpointHeight: 1 }); + genesisImport.hasSeeded.mockResolvedValue(true); + + await runner.start(); + + expect(logger.warn).not.toHaveBeenCalledWith(expect.objectContaining({ event: "GENESIS_IMPORT_SKIPPED_RESUMED_WITHOUT_MARKER" })); + }); + }); + + function setup(input: { + tipHeight: number; + archiveEnabled?: boolean; + archiveFailure?: Error; + genesisImportEnabled?: boolean; + checkpointHeight?: number; + omitStartHeight?: boolean; + }) { const archiveEnabled = input.archiveEnabled ?? true; const config = envSchema.parse({ POSTGRES_DB_URI: "postgres://unit:unit@localhost:5432/unit", - SYNC_START_HEIGHT: "1", - ARCHIVE_BUCKET: archiveEnabled ? "raw-blocks" : "" + ...(input.omitStartHeight ? {} : { SYNC_START_HEIGHT: "1" }), + ARCHIVE_BUCKET: archiveEnabled ? "raw-blocks" : "", + ...(input.genesisImportEnabled ? { GENESIS_IMPORT: "true" } : {}) }); const dbFake = { select: () => ({ - from: () => ({ - where: () => Promise.resolve([]) + from: (table: unknown) => ({ + where: () => { + if (table === IndexerState && input.checkpointHeight != null) { + return Promise.resolve([{ stream: "sync", lastHeight: input.checkpointHeight }]); + } + if (table === Blocks && input.checkpointHeight != null) { + return Promise.resolve([{ height: input.checkpointHeight, hash: Buffer.from(`hash-${input.checkpointHeight}`) }]); + } + return Promise.resolve([]); + } }) }) }; @@ -104,15 +184,16 @@ describe(SyncRunnerService.name, () => { } const committer = mock(); + const genesisImport = mock(); const logger = mock(); - const runner = new SyncRunnerService(dbFake as unknown as ChainDatabase, pool, decoder, committer, archive, config, logger); + const runner = new SyncRunnerService(dbFake as unknown as ChainDatabase, pool, decoder, committer, archive, genesisImport, config, logger); committer.commit.mockImplementation(async decoded => { if (decoded.height >= input.tipHeight) { await runner.dispose(); } }); - return { runner, archive, committer, logger, pool }; + return { runner, archive, committer, genesisImport, logger, pool }; } function buildDecodedBlock(height: number): DecodedBlock { diff --git a/apps/chain-indexer/src/pipeline/sync-runner.service.ts b/apps/chain-indexer/src/pipeline/sync-runner.service.ts index 35518c13b3..e37f38afdc 100644 --- a/apps/chain-indexer/src/pipeline/sync-runner.service.ts +++ b/apps/chain-indexer/src/pipeline/sync-runner.service.ts @@ -6,6 +6,7 @@ import { fetchRawBlock } from "@src/archive/archive-layout"; import { BlockArchiveService } from "@src/archive/block-archive.service"; import type { EnvConfig } from "@src/config/env.config"; import { Blocks, IndexerState } from "@src/db/schema"; +import { GenesisImportService } from "@src/genesis/genesis-import.service"; import { BlockCommitterService, SYNC_STREAM } from "@src/pipeline/block-committer.service"; import { BlockDecoderService } from "@src/pipeline/block-decoder.service"; import { ChainContinuityError } from "@src/pipeline/chain-continuity-error"; @@ -26,6 +27,7 @@ export class SyncRunnerService { readonly #decoder: BlockDecoderService; readonly #committer: BlockCommitterService; readonly #archive: BlockArchiveService; + readonly #genesisImport: GenesisImportService; readonly #config: EnvConfig; readonly #logger: LoggerService; @@ -38,6 +40,7 @@ export class SyncRunnerService { @inject(BlockDecoderService) decoder: BlockDecoderService, @inject(BlockCommitterService) committer: BlockCommitterService, @inject(BlockArchiveService) archive: BlockArchiveService, + @inject(GenesisImportService) genesisImport: GenesisImportService, @inject(APP_CONFIG) config: EnvConfig, @inject(LoggerService) logger: LoggerService ) { @@ -46,6 +49,7 @@ export class SyncRunnerService { this.#decoder = decoder; this.#committer = committer; this.#archive = archive; + this.#genesisImport = genesisImport; this.#config = config; this.#logger = logger; this.#logger.setContext("SYNC"); @@ -68,7 +72,13 @@ export class SyncRunnerService { } async #run(): Promise { - let nextHeight = await this.#resolveStartHeight(); + const { height, resumed } = await this.#resolveStartHeight(); + + if (this.#config.GENESIS_IMPORT) { + await this.#seedGenesisOrWarn(height, resumed); + } + + let nextHeight = height; this.#logger.info({ event: "SYNC_STARTED", network: this.#config.NETWORK, nextHeight }); this.#archive.logState(); @@ -88,6 +98,22 @@ export class SyncRunnerService { } } + /** + * Genesis is seeded only on a fresh start; a resume is already past genesis and seeding mid-chain is refused + * by design. Turning GENESIS_IMPORT on after an indexer already has a sync checkpoint would otherwise skip the + * seed with no trace, so warn when the flag is set on a resume whose genesis was never seeded. + */ + async #seedGenesisOrWarn(height: number, resumed: boolean): Promise { + if (!resumed) { + await this.#genesisImport.ensureSeeded(height); + return; + } + + if (!(await this.#genesisImport.hasSeeded())) { + this.#logger.warn({ event: "GENESIS_IMPORT_SKIPPED_RESUMED_WITHOUT_MARKER", height }); + } + } + async #retryTransient(operation: () => Promise, logContext: { event: string; height?: number }): Promise { return await retryTransient(operation, { isStopped: () => this.#stopped, logger: this.#logger, logContext }); } @@ -125,19 +151,20 @@ export class SyncRunnerService { } } - async #resolveStartHeight(): Promise { + /** `resumed` distinguishes continuing from an existing sync checkpoint from a fresh forward start, which gates whether the one-time genesis seed runs. */ + async #resolveStartHeight(): Promise<{ height: number; resumed: boolean }> { const [state] = await this.#db.select().from(IndexerState).where(eq(IndexerState.stream, SYNC_STREAM)); if (state) { const [checkpointBlock] = await this.#db.select().from(Blocks).where(eq(Blocks.height, state.lastHeight)); this.#lastHash = checkpointBlock?.hash ?? null; - return state.lastHeight + 1; + return { height: state.lastHeight + 1, resumed: true }; } if (this.#config.SYNC_START_HEIGHT) { - return this.#config.SYNC_START_HEIGHT; + return { height: this.#config.SYNC_START_HEIGHT, resumed: false }; } - return await this.#pool.getTipHeight(); + return { height: await this.#pool.getTipHeight(), resumed: false }; } } diff --git a/apps/chain-indexer/src/providers/db.provider.ts b/apps/chain-indexer/src/providers/db.provider.ts index b62480732b..092ee3e778 100644 --- a/apps/chain-indexer/src/providers/db.provider.ts +++ b/apps/chain-indexer/src/providers/db.provider.ts @@ -11,6 +11,10 @@ import { APP_CONFIG } from "@src/providers/app-config.provider"; const createDatabase = (c: DependencyContainer) => drizzle(c.resolve(PgClientService).client, { schema }); export type ChainDatabase = ReturnType; + +/** The transaction handle drizzle passes to a `db.transaction(async tx => …)` callback; lets services take a `tx` param without restating drizzle's generics. */ +export type ChainTransaction = Parameters[0]>[0]; + export const CHAIN_DB: InjectionToken = Symbol("CHAIN_DB"); container.register(CHAIN_DB, { diff --git a/apps/chain-indexer/src/rpc/rpc-client-pool.service.spec.ts b/apps/chain-indexer/src/rpc/rpc-client-pool.service.spec.ts index e506dd097d..6561c5120d 100644 --- a/apps/chain-indexer/src/rpc/rpc-client-pool.service.spec.ts +++ b/apps/chain-indexer/src/rpc/rpc-client-pool.service.spec.ts @@ -88,6 +88,16 @@ describe(RpcClientPool.name, () => { expect(fetchMock.mock.calls.map(call => call[0])).toEqual(["http://node-a/block?height=7", "http://node-a/block_results?height=7"]); }); + it("requests a genesis chunk by index", async () => { + const { pool, fetchMock } = setup(); + fetchMock.mockResolvedValue(jsonResponse({ result: { chunk: "2", total: "5", data: "eyJ9" } })); + + const chunk = await pool.getGenesisChunk(2); + + expect(chunk).toEqual({ chunk: "2", total: "5", data: "eyJ9" }); + expect(fetchMock.mock.calls[0][0]).toBe("http://node-a/genesis_chunked?chunk=2"); + }); + function setup() { const fetchMock = vi.fn(); vi.stubGlobal("fetch", fetchMock); diff --git a/apps/chain-indexer/src/rpc/rpc-client-pool.service.ts b/apps/chain-indexer/src/rpc/rpc-client-pool.service.ts index 27f76063e4..3cee43a144 100644 --- a/apps/chain-indexer/src/rpc/rpc-client-pool.service.ts +++ b/apps/chain-indexer/src/rpc/rpc-client-pool.service.ts @@ -4,7 +4,7 @@ import { inject, singleton } from "tsyringe"; import type { EnvConfig } from "@src/config/env.config"; import { APP_CONFIG } from "@src/providers/app-config.provider"; import { LoggerService } from "@src/providers/logging.provider"; -import type { RpcBlockResult, RpcBlockResultsResult, RpcStatusResult } from "@src/rpc/rpc-types"; +import type { RpcBlockResult, RpcBlockResultsResult, RpcGenesisChunkResult, RpcStatusResult } from "@src/rpc/rpc-types"; interface RpcNodeState { endpoint: string; @@ -59,6 +59,10 @@ export class RpcClientPool { return await this.#get(`/block_results?height=${height}`); } + async getGenesisChunk(chunk: number): Promise { + return await this.#get(`/genesis_chunked?chunk=${chunk}`); + } + async #get(path: string): Promise { const errors: unknown[] = []; diff --git a/apps/chain-indexer/src/rpc/rpc-types.ts b/apps/chain-indexer/src/rpc/rpc-types.ts index 7b3da76653..60fbec7b23 100644 --- a/apps/chain-indexer/src/rpc/rpc-types.ts +++ b/apps/chain-indexer/src/rpc/rpc-types.ts @@ -38,3 +38,10 @@ export interface RpcBlockResultsResult { height: string; txs_results: RpcTxResult[] | null; } + +/** CometBFT `/genesis_chunked` response. `chunk`/`total` are marshaled as strings; `data` is base64-encoded genesis JSON. */ +export interface RpcGenesisChunkResult { + chunk: string | number; + total: string | number; + data: string; +} diff --git a/apps/chain-indexer/test/fakes/build-tx-fake.ts b/apps/chain-indexer/test/fakes/build-tx-fake.ts new file mode 100644 index 0000000000..0ca740a754 --- /dev/null +++ b/apps/chain-indexer/test/fakes/build-tx-fake.ts @@ -0,0 +1,39 @@ +import type { ChainTransaction } from "@src/providers/db.provider"; + +export interface RecordedInsert { + table: unknown; + rows: Record[]; +} + +/** + * Minimal drizzle-transaction double that records inserts and supports the seeders' + * `.values(...).onConflictDoNothing().returning()` chain. `.returning()` echoes each inserted row with an + * incrementing `id`, matching how genesis seeds an empty accounts table and reads the ids straight back. + */ +export function buildTxFake(): { tx: ChainTransaction; inserts: RecordedInsert[] } { + const inserts: RecordedInsert[] = []; + let nextId = 1; + + const tx = { + insert(table: unknown) { + return { + values(rows: Record | Record[]) { + const rowArray = Array.isArray(rows) ? rows : [rows]; + inserts.push({ table, rows: rowArray }); + const returning = () => Promise.resolve(rowArray.map(row => ({ id: nextId++, ...row }))); + return Object.assign(Promise.resolve(), { + returning, + onConflictDoNothing: () => Object.assign(Promise.resolve(), { returning }), + onConflictDoUpdate: () => Promise.resolve() + }); + } + }; + } + }; + + return { tx: tx as unknown as ChainTransaction, inserts }; +} + +export function rowsFor(inserts: RecordedInsert[], table: unknown): Record[] { + return inserts.filter(insert => insert.table === table).flatMap(insert => insert.rows); +} diff --git a/apps/chain-indexer/test/fakes/genesis-fixtures.ts b/apps/chain-indexer/test/fakes/genesis-fixtures.ts new file mode 100644 index 0000000000..6c95f9bdcb --- /dev/null +++ b/apps/chain-indexer/test/fakes/genesis-fixtures.ts @@ -0,0 +1,112 @@ +import type { ParsedGenesis } from "@src/genesis/genesis-schema"; + +const VALIDATOR_OPERATOR_ADDRESS = "akashvaloper1dq9wvqemmpvanmwsdttajsn4hmtx5zk7cgw7cz"; +const VALIDATOR_ACCOUNT_ADDRESS = "akash1dq9wvqemmpvanmwsdttajsn4hmtx5zk7j2qcgg"; +const VALIDATOR_PUBKEY = "1YM8H2iPYXxzSEQeFJQipwRnWV4sB2EKgujqdeTYLJs="; +const VALIDATOR_HEX_ADDRESS = "31410FDD5FF7717918AB0D32645E12B6863B2576"; + +/** + * A small but representative Cosmos-SDK genesis document: a base account, a module account, and a + * vesting account whose balances total exactly `bank.supply`, one validator created via a genutil + * gentx, and one explicit delegation. Mirrors the sandbox-2 gentx shape verified against live RPC. + */ +export function buildRawGenesis(): Record { + return { + chain_id: "sandbox-2", + initial_height: "1", + genesis_time: "2025-10-03T17:35:37Z", + app_state: { + auth: { + accounts: [ + { "@type": "/cosmos.auth.v1beta1.BaseAccount", address: "akash1base", account_number: "1", sequence: "0" }, + { + "@type": "/cosmos.auth.v1beta1.ModuleAccount", + base_account: { address: "akash1module", account_number: "2", sequence: "0" }, + name: "bonded_tokens_pool", + permissions: [] + }, + { + "@type": "/cosmos.vesting.v1beta1.ContinuousVestingAccount", + base_vesting_account: { + base_account: { address: "akash1vesting", account_number: "3", sequence: "0" }, + original_vesting: [{ denom: "uakt", amount: "20" }] + }, + start_time: "0" + } + ] + }, + bank: { + balances: [ + { address: "akash1base", coins: [{ denom: "uakt", amount: "10" }] }, + { address: "akash1module", coins: [{ denom: "uakt", amount: "5" }] }, + { address: "akash1vesting", coins: [{ denom: "uakt", amount: "20" }] } + ], + supply: [{ denom: "uakt", amount: "35" }] + }, + staking: { + params: { bond_denom: "uakt" }, + validators: [], + delegations: [{ delegator_address: "akash1base", validator_address: VALIDATOR_OPERATOR_ADDRESS, shares: "1000000.000000000000000000" }] + }, + genutil: { + gen_txs: [ + { + body: { + messages: [ + { + "@type": "/cosmos.staking.v1beta1.MsgCreateValidator", + description: { moniker: "validator-01", identity: "", website: "", security_contact: "", details: "" }, + commission: { rate: "0.100000000000000000", max_rate: "0.200000000000000000", max_change_rate: "0.010000000000000000" }, + min_self_delegation: "1", + delegator_address: VALIDATOR_ACCOUNT_ADDRESS, + validator_address: VALIDATOR_OPERATOR_ADDRESS, + pubkey: { "@type": "/cosmos.crypto.ed25519.PubKey", key: VALIDATOR_PUBKEY }, + value: { denom: "uakt", amount: "1000000" } + } + ] + } + } + ] + } + } + }; +} + +/** The exact `ParsedGenesis` that `parseGenesis(buildRawGenesis())` must produce. */ +export function buildParsedGenesis(): ParsedGenesis { + return { + chainId: "sandbox-2", + initialHeight: 1, + genesisTime: "2025-10-03T17:35:37Z", + bondDenom: "uakt", + accounts: [ + { address: "akash1base", accountNumber: 1, accountType: "base", isModuleAccount: false }, + { address: "akash1module", accountNumber: 2, accountType: "module", isModuleAccount: true }, + { address: "akash1vesting", accountNumber: 3, accountType: "vesting", isModuleAccount: false } + ], + unknownAccountTypes: [], + balances: [ + { address: "akash1base", coins: [{ denom: "uakt", amount: "10" }] }, + { address: "akash1module", coins: [{ denom: "uakt", amount: "5" }] }, + { address: "akash1vesting", coins: [{ denom: "uakt", amount: "20" }] } + ], + supply: [{ denom: "uakt", amount: "35" }], + validators: [ + { + operatorAddress: VALIDATOR_OPERATOR_ADDRESS, + accountAddress: VALIDATOR_ACCOUNT_ADDRESS, + hexAddress: VALIDATOR_HEX_ADDRESS, + moniker: "validator-01", + identity: "", + website: "", + details: "", + securityContact: "", + commissionRate: "0.100000000000000000", + commissionMaxRate: "0.200000000000000000", + commissionMaxChangeRate: "0.010000000000000000", + minSelfDelegation: "1" + } + ], + delegations: [{ delegatorAddress: "akash1base", validatorOperatorAddress: VALIDATOR_OPERATOR_ADDRESS, shares: "1000000.000000000000000000" }] + }; +} From 35cfc61c09aedd1d80eabb64d16a0c234ea12046 Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Thu, 13 Aug 2026 23:26:31 +0530 Subject: [PATCH 06/14] feat(indexer): balance ledger and address activity log (CON-809) (#3588) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(indexer): capture block/tx events and ledger idempotency schema Add the L-4 write-path foundation: a 0002 migration adding the (height, event_index) idempotency key and tx_index to balance_changes, the account_txs address-activity table, and a staking balance reason. The block decoder now captures coin/transfer/mint/burn/slash events with base64-normalized attributes and derives per-tx signer addresses, the enabler for deriving balances from events. Genesis ledger rows move to initialHeight-1 so they never collide with block initialHeight's events. * feat(indexer): derive balance changes from coin events Add the pure, DB-free derivation layer: parseCoins for coin strings, a module-address registry deriving Cosmos SDK module accounts by name (sha256(name)[:20]), an MVP reason classifier, and the balance deriver that turns coin_spent/coin_received into ordered movements with a deterministic block-wide event_index and correlated counterparties. * feat(indexer): intern accounts and write the balance ledger idempotently Add the account interner (resolve-or-create, no permanent cache) and the balance writer. The writer seeds each running balance from the ledger baseline (not the snapshot), appends balance_changes on the (height, event_index) unique key, and advances account_balances additively by only the rows it actually inserted — so overlapping sync/backfill writers apply every delta exactly once and re-commits are no-ops. * feat(indexer): write balance ledger and activity log per block Wire balance derivation into the block committer: derive changes and address-activity rows per block, intern all touched addresses on the base connection, then inside the commit transaction write the balance ledger and account_txs after messages and before the checkpoint. Both sync and backfill flow through commitBatch, so both persist the ledger. * feat(indexer): add reconciliation script proving ledger matches chain Add abci_query historical support to the RPC pool and a one-shot npm run reconcile entrypoint: at the sync checkpoint height it compares each sampled account's current balance against the node's bank balance and the ledger's per-denom totals against chain supply, exiting non-zero on any mismatch so it can gate a deploy. * fix(indexer): correct balance reason classification and reconcile guard Scope the slash reason to the coincident burn leg so a validator slash no longer mislabels the block's routine inflation mint (and other block-level movements sharing the scope) as slash. Add the missing mint case to the reason classifier so a mint-module counterparty resolves to mint instead of falling through to transfer. Reject a non-numeric RECONCILE_SAMPLE_SIZE instead of coercing it to NaN, which slice() turned into an empty sample that could report a clean pass having verified zero accounts. Drop the unread "message" event type from the decoder whitelist and remove a dead branch in decodeIfBase64. * fix(indexer): classify escrow module movements as escrow The reason classifier fell through to `transfer` for any coin movement involving the x/escrow module account, because that module wasn't in the address registry. It mislabeled every ACT (uact) flow — which only moves through escrow, never peer-to-peer — plus uakt escrow deposits, refunds, and lease settlements, as generic transfers. Add the escrow module to the registry and map it to the existing `escrow` reason. Verified against a mainnet day of blocks: uact movements now classify as escrow (previously 100% transfer) while per-account and total supply balances still reconcile exactly to the chain. * refactor(indexer): use insertChunked for block, transaction and message inserts * fix(indexer): scope slash reason, correlate counterparty by amount, fail over abci non-zero code * fix(indexer): reconcile from a consistent snapshot with bounded concurrency Read the checkpoint height and ledger balances inside one REPEATABLE READ read-only transaction so a concurrent block commit can't pair a stale height with post-commit balances and flag spurious mismatches. Fan out the per-account bank queries with bounded concurrency instead of one sequential RPC round-trip at a time, letting the RpcClientPool load-balance the sampled accounts. * fix(indexer): classify balance reason by the holder's own module role first For a module-to-module movement (fee_collector to distribution, mint to fee_collector) both sides carry a role, so preferring the counterparty tagged each leg with the other module's reason. Prefer the holder's own role and fall back to the counterparty, so every leg is classified by the module whose balance actually changed. Account-to-module movements are unaffected: the account side has no role and still falls through to the counterparty. * fix(indexer): chunk account existence lookup and document ledger Chunk the account existence SELECT the same way the insert path already does, so a batch with tens of thousands of distinct addresses can't exceed postgres.js's bind-parameter limit and abort the commit transaction. Document the balance ledger, activity log, and the reconcile script in the app README alongside the existing feature sections. * refactor(indexer): compute each account's total once when sampling Precompute the coin total per account before sorting the reconciliation sample instead of re-reducing both operands on every comparator call. * fix(indexer): chunk balance baseline lookup under the bind-parameter limit #readLedgerBaseline built accountIds/denoms from a whole commitBatch's intents and passed them into one unchunked inArray SELECT, unlike #insertChanges/#applyNetDeltas in the same class. A dense backfill batch near BACKFILL_BATCH_SIZE's 1000-block max can touch tens of thousands of distinct accounts, exceeding postgres.js's ~65k bind-parameter cap and aborting the whole commitBatch transaction. Chunk accountIds by INSERT_CHUNK_SIZE and merge per-chunk rows, matching AccountInterner's already-chunked select. * refactor(indexer): validate RECONCILE_SAMPLE_SIZE through the env schema The reconcile CLI hand-rolled Number() parsing plus a bespoke Number.isInteger/positive check and its own CONFIG_INVALID log shape, duplicating the zod envSchema every other numeric env var already flows through. Add RECONCILE_SAMPLE_SIZE to envSchema as an optional coerced positive int (same idiom as SYNC_START_HEIGHT) and read it from parsed.data, so an invalid value fails at the existing safeParse path. The service keeps its own guard as defense in depth. --- apps/chain-indexer/README.md | 10 + .../drizzle/0002_clever_bulldozer.sql | 14 + .../drizzle/meta/0002_snapshot.json | 801 ++++++++++++++++++ apps/chain-indexer/drizzle/meta/_journal.json | 7 + apps/chain-indexer/package.json | 2 + .../src/config/env.config.spec.ts | 20 + apps/chain-indexer/src/config/env.config.ts | 2 + apps/chain-indexer/src/db/schema.spec.ts | 20 +- apps/chain-indexer/src/db/schema.ts | 30 +- .../src/genesis/bank-seeder.service.spec.ts | 6 +- .../src/genesis/bank-seeder.service.ts | 9 +- .../src/genesis/genesis-address.ts | 3 + .../balance/account-interner.service.spec.ts | 108 +++ .../balance/account-interner.service.ts | 64 ++ .../balance/account-tx-deriver.spec.ts | 92 ++ .../pipeline/balance/account-tx-deriver.ts | 49 ++ .../pipeline/balance/balance-deriver.spec.ts | 230 +++++ .../src/pipeline/balance/balance-deriver.ts | 154 ++++ .../balance/balance-writer.service.spec.ts | 171 ++++ .../balance/balance-writer.service.ts | 152 ++++ .../src/pipeline/balance/coin-amount.spec.ts | 35 + .../src/pipeline/balance/coin-amount.ts | 24 + .../balance/module-address-registry.spec.ts | 38 + .../balance/module-address-registry.ts | 58 ++ .../balance/reason-classifier.spec.ts | 101 +++ .../src/pipeline/balance/reason-classifier.ts | 60 ++ .../pipeline/block-committer.service.spec.ts | 82 +- .../src/pipeline/block-committer.service.ts | 87 +- .../pipeline/block-decoder.service.spec.ts | 98 ++- .../src/pipeline/block-decoder.service.ts | 52 +- .../src/pipeline/decode-if-base64.spec.ts | 29 + .../src/pipeline/decode-if-base64.ts | 35 + .../src/pipeline/decoded-block.ts | 10 + .../src/pipeline/signer-addresses.spec.ts | 38 + .../src/pipeline/signer-addresses.ts | 44 + .../src/reconcile/bank-query.spec.ts | 44 + .../chain-indexer/src/reconcile/bank-query.ts | 40 + .../src/reconcile/coin-diff.spec.ts | 33 + apps/chain-indexer/src/reconcile/coin-diff.ts | 30 + .../src/reconcile/reconcile.service.spec.ts | 132 +++ .../src/reconcile/reconcile.service.ts | 167 ++++ apps/chain-indexer/src/reconcile/reconcile.ts | 37 + .../src/rpc/rpc-client-pool.service.spec.ts | 36 + .../src/rpc/rpc-client-pool.service.ts | 26 +- apps/chain-indexer/src/rpc/rpc-types.ts | 20 + apps/chain-indexer/tsup.config.ts | 1 + package-lock.json | 68 ++ 47 files changed, 3331 insertions(+), 38 deletions(-) create mode 100644 apps/chain-indexer/drizzle/0002_clever_bulldozer.sql create mode 100644 apps/chain-indexer/drizzle/meta/0002_snapshot.json create mode 100644 apps/chain-indexer/src/pipeline/balance/account-interner.service.spec.ts create mode 100644 apps/chain-indexer/src/pipeline/balance/account-interner.service.ts create mode 100644 apps/chain-indexer/src/pipeline/balance/account-tx-deriver.spec.ts create mode 100644 apps/chain-indexer/src/pipeline/balance/account-tx-deriver.ts create mode 100644 apps/chain-indexer/src/pipeline/balance/balance-deriver.spec.ts create mode 100644 apps/chain-indexer/src/pipeline/balance/balance-deriver.ts create mode 100644 apps/chain-indexer/src/pipeline/balance/balance-writer.service.spec.ts create mode 100644 apps/chain-indexer/src/pipeline/balance/balance-writer.service.ts create mode 100644 apps/chain-indexer/src/pipeline/balance/coin-amount.spec.ts create mode 100644 apps/chain-indexer/src/pipeline/balance/coin-amount.ts create mode 100644 apps/chain-indexer/src/pipeline/balance/module-address-registry.spec.ts create mode 100644 apps/chain-indexer/src/pipeline/balance/module-address-registry.ts create mode 100644 apps/chain-indexer/src/pipeline/balance/reason-classifier.spec.ts create mode 100644 apps/chain-indexer/src/pipeline/balance/reason-classifier.ts create mode 100644 apps/chain-indexer/src/pipeline/decode-if-base64.spec.ts create mode 100644 apps/chain-indexer/src/pipeline/decode-if-base64.ts create mode 100644 apps/chain-indexer/src/pipeline/signer-addresses.spec.ts create mode 100644 apps/chain-indexer/src/pipeline/signer-addresses.ts create mode 100644 apps/chain-indexer/src/reconcile/bank-query.spec.ts create mode 100644 apps/chain-indexer/src/reconcile/bank-query.ts create mode 100644 apps/chain-indexer/src/reconcile/coin-diff.spec.ts create mode 100644 apps/chain-indexer/src/reconcile/coin-diff.ts create mode 100644 apps/chain-indexer/src/reconcile/reconcile.service.spec.ts create mode 100644 apps/chain-indexer/src/reconcile/reconcile.service.ts create mode 100644 apps/chain-indexer/src/reconcile/reconcile.ts diff --git a/apps/chain-indexer/README.md b/apps/chain-indexer/README.md index 1e2d450838..0f034b3acc 100644 --- a/apps/chain-indexer/README.md +++ b/apps/chain-indexer/README.md @@ -58,6 +58,16 @@ BACKFILL_TO_HEIGHT=200000 Blocks are fetched from RPC in parallel (`BACKFILL_CONCURRENCY`, default 10) and committed strictly in order in batches of `BACKFILL_BATCH_SIZE` blocks (default 200), each batch in one Postgres transaction together with the checkpoint advance. Progress is checkpointed per range under the `indexer_state` stream `backfill:{from}-{to}`, so killing and restarting the job resumes at the checkpoint without gaps or duplicates, and re-running a completed range exits 0 immediately. Changing the range creates a fresh checkpoint row. All inserts are natural-keyed and conflict-ignoring, so a backfill can run against the same database as live sync, and a duplicate backfill pod on the same range is harmless. +## Balance ledger and activity log + +Every committed block also derives a balance ledger and an address activity log, in the same transaction as the block, so they never drift from the chain data they come from. `balance_changes` is the append-only ledger: one row per coin movement with the running `balance_after` and a classified `reason` (`mint`, `burn`, `slash`, `fee`, `reward`, `commission`, `staking`, `gov`, `ibc`, `escrow`, `bme`, or a plain `transfer`; genesis seeds are `genesis`). `account_balances` holds the current per-account per-denom balance, upserted from the ledger. `account_txs` is the activity log linking each account to the transactions that touched it. Addresses are interned to ids on first sight (`accounts`), so both live sync and backfill produce identical ledger rows for the same height. + +The reason heuristic is deliberately MVP: coincident mint/burn/slash win first, then the module account on the holder's side of the movement (falling back to the counterparty's), then the denom. Per-deployment/lease attribution of escrow movements is left for later. + +## Reconciliation + +`npm run reconcile` proves the ledger matches the chain at the `sync` checkpoint height. It samples the highest-balance accounts, compares each against the node's bank balance at that height, and checks the ledger's per-denom totals against the chain's total supply; it exits non-zero on any mismatch or misconfiguration, so it can gate a deploy. Querying at the checkpoint rather than the moving tip keeps the comparison race-free, which requires an unpruned (archival) node — sandbox is archival. `RECONCILE_SAMPLE_SIZE` overrides the default sample of 100 accounts. + ## Raw block archive Set `ARCHIVE_BUCKET` to a GCS bucket name to keep a zstd-compressed copy of every raw `/block` and `/block_results` payload, so handler fixes and new modules can be replayed without re-fetching history from RPC. Leave it unset and both roles behave exactly as before (the boot log says `ARCHIVE_DISABLED`). Authentication uses Application Default Credentials; no key material is configured in the app. diff --git a/apps/chain-indexer/drizzle/0002_clever_bulldozer.sql b/apps/chain-indexer/drizzle/0002_clever_bulldozer.sql new file mode 100644 index 0000000000..1c03820219 --- /dev/null +++ b/apps/chain-indexer/drizzle/0002_clever_bulldozer.sql @@ -0,0 +1,14 @@ +CREATE TYPE "cosmos"."account_tx_role" AS ENUM('signer', 'sender', 'receiver');--> statement-breakpoint +ALTER TYPE "cosmos"."balance_change_reason" ADD VALUE 'staking';--> statement-breakpoint +CREATE TABLE "cosmos"."account_txs" ( + "account_id" integer NOT NULL, + "height" bigint NOT NULL, + "tx_index" integer NOT NULL, + "role" "cosmos"."account_tx_role" NOT NULL, + CONSTRAINT "account_txs_account_id_height_tx_index_role_pk" PRIMARY KEY("account_id","height","tx_index","role") +); +--> statement-breakpoint +ALTER TABLE "cosmos"."balance_changes" ADD COLUMN "tx_index" integer;--> statement-breakpoint +ALTER TABLE "cosmos"."balance_changes" ADD COLUMN "event_index" integer NOT NULL;--> statement-breakpoint +ALTER TABLE "cosmos"."account_txs" ADD CONSTRAINT "account_txs_account_id_accounts_id_fk" FOREIGN KEY ("account_id") REFERENCES "cosmos"."accounts"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +CREATE UNIQUE INDEX "balance_changes_height_event_index_idx" ON "cosmos"."balance_changes" USING btree ("height","event_index"); \ No newline at end of file diff --git a/apps/chain-indexer/drizzle/meta/0002_snapshot.json b/apps/chain-indexer/drizzle/meta/0002_snapshot.json new file mode 100644 index 0000000000..423c04e22e --- /dev/null +++ b/apps/chain-indexer/drizzle/meta/0002_snapshot.json @@ -0,0 +1,801 @@ +{ + "id": "b507e1c8-31e3-46c8-8dc2-c0c6927e972d", + "prevId": "05402ce9-8c3f-4514-8150-e7911053df30", + "version": "7", + "dialect": "postgresql", + "tables": { + "cosmos.account_balances": { + "name": "account_balances", + "schema": "cosmos", + "columns": { + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "amount": { + "name": "amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "account_balances_account_id_accounts_id_fk": { + "name": "account_balances_account_id_accounts_id_fk", + "tableFrom": "account_balances", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_balances_account_id_denom_pk": { + "name": "account_balances_account_id_denom_pk", + "columns": [ + "account_id", + "denom" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.account_txs": { + "name": "account_txs", + "schema": "cosmos", + "columns": { + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "account_tx_role", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "account_txs_account_id_accounts_id_fk": { + "name": "account_txs_account_id_accounts_id_fk", + "tableFrom": "account_txs", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_txs_account_id_height_tx_index_role_pk": { + "name": "account_txs_account_id_height_tx_index_role_pk", + "columns": [ + "account_id", + "height", + "tx_index", + "role" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.accounts": { + "name": "accounts", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_number": { + "name": "account_number", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "account_type": { + "name": "account_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_module_account": { + "name": "is_module_account", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": { + "accounts_address_idx": { + "name": "accounts_address_idx", + "columns": [ + { + "expression": "address", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.balance_changes": { + "name": "balance_changes", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delta": { + "name": "delta", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "balance_after": { + "name": "balance_after", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "balance_change_reason", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "event_index": { + "name": "event_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "counterparty_account_id": { + "name": "counterparty_account_id", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "balance_changes_account_denom_height_idx": { + "name": "balance_changes_account_denom_height_idx", + "columns": [ + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "denom", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "balance_changes_height_event_index_idx": { + "name": "balance_changes_height_event_index_idx", + "columns": [ + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_index", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "balance_changes_account_id_accounts_id_fk": { + "name": "balance_changes_account_id_accounts_id_fk", + "tableFrom": "balance_changes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "balance_changes_counterparty_account_id_accounts_id_fk": { + "name": "balance_changes_counterparty_account_id_accounts_id_fk", + "tableFrom": "balance_changes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "counterparty_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.blocks": { + "name": "blocks", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "datetime": { + "name": "datetime", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "parent_hash": { + "name": "parent_hash", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "proposer_address": { + "name": "proposer_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tx_count": { + "name": "tx_count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.delegations": { + "name": "delegations", + "schema": "cosmos", + "columns": { + "delegator_account_id": { + "name": "delegator_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "validator_operator_address": { + "name": "validator_operator_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "shares": { + "name": "shares", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "delegations_delegator_account_id_accounts_id_fk": { + "name": "delegations_delegator_account_id_accounts_id_fk", + "tableFrom": "delegations", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "delegator_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "delegations_delegator_account_id_validator_operator_address_pk": { + "name": "delegations_delegator_account_id_validator_operator_address_pk", + "columns": [ + "delegator_account_id", + "validator_operator_address" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.indexer_state": { + "name": "indexer_state", + "schema": "", + "columns": { + "stream": { + "name": "stream", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "last_height": { + "name": "last_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.message_types": { + "name": "message_types", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "message_types_type_idx": { + "name": "message_types_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.messages": { + "name": "messages", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type_id": { + "name": "type_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "body": { + "name": "body", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "messages_type_id_idx": { + "name": "messages_type_id_idx", + "columns": [ + { + "expression": "type_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "messages_type_id_message_types_id_fk": { + "name": "messages_type_id_message_types_id_fk", + "tableFrom": "messages", + "tableTo": "message_types", + "schemaTo": "cosmos", + "columnsFrom": [ + "type_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messages_height_tx_index_index_pk": { + "name": "messages_height_tx_index_index_pk", + "columns": [ + "height", + "tx_index", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.transactions": { + "name": "transactions", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "code": { + "name": "code", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "gas_used": { + "name": "gas_used", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gas_wanted": { + "name": "gas_wanted", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "fee": { + "name": "fee", + "type": "jsonb", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "transactions_hash_idx": { + "name": "transactions_hash_idx", + "columns": [ + { + "expression": "hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "transactions_height_index_pk": { + "name": "transactions_height_index_pk", + "columns": [ + "height", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.validators": { + "name": "validators", + "schema": "cosmos", + "columns": { + "operator_address": { + "name": "operator_address", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_address": { + "name": "account_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "hex_address": { + "name": "hex_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "moniker": { + "name": "moniker", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "identity": { + "name": "identity", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "details": { + "name": "details", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "security_contact": { + "name": "security_contact", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commission_rate": { + "name": "commission_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "commission_max_rate": { + "name": "commission_max_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "commission_max_change_rate": { + "name": "commission_max_change_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "min_self_delegation": { + "name": "min_self_delegation", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "cosmos.account_tx_role": { + "name": "account_tx_role", + "schema": "cosmos", + "values": [ + "signer", + "sender", + "receiver" + ] + }, + "cosmos.balance_change_reason": { + "name": "balance_change_reason", + "schema": "cosmos", + "values": [ + "genesis", + "transfer", + "fee", + "reward", + "commission", + "slash", + "gov", + "ibc", + "escrow", + "bme", + "mint", + "burn", + "staking" + ] + } + }, + "schemas": { + "cosmos": "cosmos" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/chain-indexer/drizzle/meta/_journal.json b/apps/chain-indexer/drizzle/meta/_journal.json index d80e14e60d..43ea741cff 100644 --- a/apps/chain-indexer/drizzle/meta/_journal.json +++ b/apps/chain-indexer/drizzle/meta/_journal.json @@ -15,6 +15,13 @@ "when": 1786548202033, "tag": "0001_long_mach_iv", "breakpoints": true + }, + { + "idx": 2, + "version": "7", + "when": 1786566671632, + "tag": "0002_clever_bulldozer", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/chain-indexer/package.json b/apps/chain-indexer/package.json index 17328b775a..6066df516e 100644 --- a/apps/chain-indexer/package.json +++ b/apps/chain-indexer/package.json @@ -13,6 +13,7 @@ "lint": "eslint .", "migration:gen": "drizzle-kit generate", "prod": "node --enable-source-maps --require ./dist/instrumentation.js ./dist/server.js", + "reconcile": "npm run build && node --enable-source-maps ./dist/reconcile.js", "start": "tsup --watch", "test": "vitest run --project unit", "test:cov": "vitest run --project unit --coverage", @@ -24,6 +25,7 @@ "@akashnetwork/instrumentation": "*", "@akashnetwork/logging": "*", "@akashnetwork/net": "*", + "@cosmjs/amino": "~0.38.0", "@cosmjs/encoding": "~0.38.0", "@cosmjs/proto-signing": "~0.38.0", "@cosmjs/stargate": "~0.38.0", diff --git a/apps/chain-indexer/src/config/env.config.spec.ts b/apps/chain-indexer/src/config/env.config.spec.ts index f448a6e043..edc4274aa7 100644 --- a/apps/chain-indexer/src/config/env.config.spec.ts +++ b/apps/chain-indexer/src/config/env.config.spec.ts @@ -81,6 +81,26 @@ describe("envSchema", () => { expect(config.BACKFILL_TO_HEIGHT).toBeUndefined(); }); + it("treats an empty RECONCILE_SAMPLE_SIZE as absent", () => { + const config = setup({ RECONCILE_SAMPLE_SIZE: "" }); + + expect(config.RECONCILE_SAMPLE_SIZE).toBeUndefined(); + }); + + it("coerces a numeric RECONCILE_SAMPLE_SIZE string", () => { + const config = setup({ RECONCILE_SAMPLE_SIZE: "250" }); + + expect(config.RECONCILE_SAMPLE_SIZE).toBe(250); + }); + + it("rejects a non-positive RECONCILE_SAMPLE_SIZE", () => { + expect(() => setup({ RECONCILE_SAMPLE_SIZE: "0" })).toThrow(); + }); + + it("rejects a fractional RECONCILE_SAMPLE_SIZE", () => { + expect(() => setup({ RECONCILE_SAMPLE_SIZE: "10.5" })).toThrow(); + }); + function setup(overrides?: Record) { return envSchema.parse({ POSTGRES_DB_URI: "postgres://unit:unit@localhost:5432/unit", ...overrides }); } diff --git a/apps/chain-indexer/src/config/env.config.ts b/apps/chain-indexer/src/config/env.config.ts index 6f3c849257..c62d647a5d 100644 --- a/apps/chain-indexer/src/config/env.config.ts +++ b/apps/chain-indexer/src/config/env.config.ts @@ -39,6 +39,8 @@ const rawEnvSchema = z.object({ ARCHIVE_STORAGE_API_ENDPOINT: z.preprocess(emptyStringAsUndefined, z.string().url().optional()), /** Decoded message bodies above this serialized size are stored as null to keep pathological messages out of Postgres. */ MESSAGE_BODY_MAX_BYTES: z.number({ coerce: true }).int().positive().default(65_536), + /** How many of the highest-balance accounts `npm run reconcile` checks against the chain. Unset defers to the service default. */ + RECONCILE_SAMPLE_SIZE: z.preprocess(emptyStringAsUndefined, z.number({ coerce: true }).int().positive().optional()), DRIZZLE_MIGRATIONS_FOLDER: z.string().default("./drizzle"), LOG_LEVEL: z.enum(["fatal", "error", "warn", "info", "debug", "trace"]).optional().default("info"), STD_OUT_LOG_FORMAT: z.enum(["json", "pretty"]).optional().default("json"), diff --git a/apps/chain-indexer/src/db/schema.spec.ts b/apps/chain-indexer/src/db/schema.spec.ts index 80c7fe9dce..a804cf4e13 100644 --- a/apps/chain-indexer/src/db/schema.spec.ts +++ b/apps/chain-indexer/src/db/schema.spec.ts @@ -1,7 +1,7 @@ import { getTableConfig } from "drizzle-orm/pg-core"; import { describe, expect, it } from "vitest"; -import { AccountBalances, Accounts, BalanceChanges, Delegations, Validators } from "@src/db/schema"; +import { AccountBalances, Accounts, AccountTxs, BalanceChanges, Delegations, Validators } from "@src/db/schema"; describe("cosmos genesis schema", () => { it("interns accounts under a unique address index", () => { @@ -25,10 +25,26 @@ describe("cosmos genesis schema", () => { const config = getTableConfig(BalanceChanges); expect(config.foreignKeys).toHaveLength(2); - expect(config.indexes).toHaveLength(1); config.foreignKeys.forEach(foreignKey => expect(foreignKey.reference().foreignColumns[0].name).toBe("id")); }); + it("makes the ledger idempotent with a unique (height, event_index) index", () => { + const config = getTableConfig(BalanceChanges); + + const uniqueOnHeightEvent = config.indexes.find(index => index.config.name === "balance_changes_height_event_index_idx"); + expect(uniqueOnHeightEvent?.config.unique).toBe(true); + expect(uniqueOnHeightEvent?.config.columns.map(column => (column as { name: string }).name)).toEqual(["height", "event_index"]); + }); + + it("keys the address activity log by account, height, tx and role", () => { + const config = getTableConfig(AccountTxs); + + expect(config.name).toBe("account_txs"); + expect(config.primaryKeys[0].columns.map(column => column.name)).toEqual(["account_id", "height", "tx_index", "role"]); + expect(config.foreignKeys).toHaveLength(1); + expect(config.foreignKeys[0].reference().foreignColumns[0].name).toBe("id"); + }); + it("keys delegations by delegator and validator with a delegator foreign key", () => { const config = getTableConfig(Delegations); diff --git a/apps/chain-indexer/src/db/schema.ts b/apps/chain-indexer/src/db/schema.ts index 93e9d6e184..19523cbb7d 100644 --- a/apps/chain-indexer/src/db/schema.ts +++ b/apps/chain-indexer/src/db/schema.ts @@ -93,7 +93,8 @@ export const balanceChangeReason = cosmosSchema.enum("balance_change_reason", [ "escrow", "bme", "mint", - "burn" + "burn", + "staking" ]); /** Addresses interned once and referenced by integer id, mirroring the message_types lookup. */ @@ -135,9 +136,14 @@ export const BalanceChanges = cosmosSchema.table( balanceAfter: numeric("balance_after", { precision: 38, scale: 0 }).notNull(), reason: balanceChangeReason("reason").notNull(), height: bigint("height", { mode: "number" }).notNull(), + txIndex: integer("tx_index"), + eventIndex: integer("event_index").notNull(), counterpartyAccountId: integer("counterparty_account_id").references(() => Accounts.id) }, - t => [index("balance_changes_account_denom_height_idx").on(t.accountId, t.denom, t.height)] + t => [ + index("balance_changes_account_denom_height_idx").on(t.accountId, t.denom, t.height), + uniqueIndex("balance_changes_height_event_index_idx").on(t.height, t.eventIndex) + ] ); export const Validators = cosmosSchema.table("validators", { @@ -166,3 +172,23 @@ export const Delegations = cosmosSchema.table( }, t => [primaryKey({ columns: [t.delegatorAccountId, t.validatorOperatorAddress] })] ); + +/** How an address participated in a transaction: it signed it, or it was the sender/recipient of a coin movement. */ +export const accountTxRole = cosmosSchema.enum("account_tx_role", ["signer", "sender", "receiver"]); + +/** + * Address activity log: one row per (address, tx, role). The leading `(accountId, height)` of the primary + * key serves "list an address's activity newest-first"; the composite key makes re-committing a block idempotent. + */ +export const AccountTxs = cosmosSchema.table( + "account_txs", + { + accountId: integer("account_id") + .notNull() + .references(() => Accounts.id), + height: bigint("height", { mode: "number" }).notNull(), + txIndex: integer("tx_index").notNull(), + role: accountTxRole("role").notNull() + }, + t => [primaryKey({ columns: [t.accountId, t.height, t.txIndex, t.role] })] +); diff --git a/apps/chain-indexer/src/genesis/bank-seeder.service.spec.ts b/apps/chain-indexer/src/genesis/bank-seeder.service.spec.ts index 6774f23516..a4fc80d4a7 100644 --- a/apps/chain-indexer/src/genesis/bank-seeder.service.spec.ts +++ b/apps/chain-indexer/src/genesis/bank-seeder.service.spec.ts @@ -18,9 +18,9 @@ describe(BankSeeder.name, () => { { accountId: 3, denom: "uakt", amount: "20" } ]); expect(rowsFor(inserts, BalanceChanges)).toEqual([ - { accountId: 1, denom: "uakt", delta: "10", balanceAfter: "10", reason: "genesis", height: 1, counterpartyAccountId: null }, - { accountId: 2, denom: "uakt", delta: "5", balanceAfter: "5", reason: "genesis", height: 1, counterpartyAccountId: null }, - { accountId: 3, denom: "uakt", delta: "20", balanceAfter: "20", reason: "genesis", height: 1, counterpartyAccountId: null } + { accountId: 1, denom: "uakt", delta: "10", balanceAfter: "10", reason: "genesis", height: 0, txIndex: null, eventIndex: 0, counterpartyAccountId: null }, + { accountId: 2, denom: "uakt", delta: "5", balanceAfter: "5", reason: "genesis", height: 0, txIndex: null, eventIndex: 1, counterpartyAccountId: null }, + { accountId: 3, denom: "uakt", delta: "20", balanceAfter: "20", reason: "genesis", height: 0, txIndex: null, eventIndex: 2, counterpartyAccountId: null } ]); }); diff --git a/apps/chain-indexer/src/genesis/bank-seeder.service.ts b/apps/chain-indexer/src/genesis/bank-seeder.service.ts index 5b27d5f690..7f803d6c1e 100644 --- a/apps/chain-indexer/src/genesis/bank-seeder.service.ts +++ b/apps/chain-indexer/src/genesis/bank-seeder.service.ts @@ -13,10 +13,15 @@ export class BankSeeder implements GenesisModuleSeeder { * Seeding all `bank.balances` (module and vesting accounts included) makes the current-balance total * reconcile to `bank.supply` by construction. Idempotency comes from the import marker, so the ledger * insert intentionally has no conflict target. + * + * The ledger rows sit at `initialHeight - 1` (the pre-block opening balance) so they never collide with + * block `initialHeight`'s own coin events on the `(height, event_index)` unique key and give that block's + * batch a correct running-balance baseline. */ async seed(tx: ChainTransaction, genesis: ParsedGenesis, context: GenesisSeedContext): Promise { const balanceRows: (typeof AccountBalances.$inferInsert)[] = []; const changeRows: (typeof BalanceChanges.$inferInsert)[] = []; + let eventIndex = 0; for (const balance of genesis.balances) { const accountId = context.accountIdByAddress.get(balance.address); @@ -32,7 +37,9 @@ export class BankSeeder implements GenesisModuleSeeder { delta: coin.amount, balanceAfter: coin.amount, reason: "genesis", - height: context.initialHeight, + height: context.initialHeight - 1, + txIndex: null, + eventIndex: eventIndex++, counterpartyAccountId: null }); } diff --git a/apps/chain-indexer/src/genesis/genesis-address.ts b/apps/chain-indexer/src/genesis/genesis-address.ts index bca386e366..c758d71c6d 100644 --- a/apps/chain-indexer/src/genesis/genesis-address.ts +++ b/apps/chain-indexer/src/genesis/genesis-address.ts @@ -1,6 +1,9 @@ import { fromBase64, fromBech32, toBech32, toHex } from "@cosmjs/encoding"; import { createHash } from "node:crypto"; +/** Bech32 human-readable prefix for account addresses. Shared by mainnet, sandbox and testnet — all Akash chains. */ +export const AKASH_ADDRESS_PREFIX = "akash"; + const ED25519_PUBKEY_TYPE = "/cosmos.crypto.ed25519.PubKey"; /** diff --git a/apps/chain-indexer/src/pipeline/balance/account-interner.service.spec.ts b/apps/chain-indexer/src/pipeline/balance/account-interner.service.spec.ts new file mode 100644 index 0000000000..e770e6f213 --- /dev/null +++ b/apps/chain-indexer/src/pipeline/balance/account-interner.service.spec.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from "vitest"; + +import { INSERT_CHUNK_SIZE } from "@src/db/insert-chunk-size"; +import { Accounts } from "@src/db/schema"; +import { AccountInterner } from "@src/pipeline/balance/account-interner.service"; +import type { ChainDatabase } from "@src/providers/db.provider"; + +describe(AccountInterner.name, () => { + it("returns existing ids without inserting when every address is already interned", async () => { + const { interner, insertedRows } = setup({ + selectResults: [ + [ + { id: 1, address: "akash1a" }, + { id: 2, address: "akash1b" } + ] + ] + }); + + const ids = await interner.resolve(["akash1a", "akash1b"]); + + expect(ids).toEqual( + new Map([ + ["akash1a", 1], + ["akash1b", 2] + ]) + ); + expect(insertedRows).toEqual([]); + }); + + it("inserts only the missing addresses and merges the returned ids", async () => { + const { interner, insertedRows } = setup({ selectResults: [[{ id: 1, address: "akash1a" }]], insertReturning: [{ id: 2, address: "akash1b" }] }); + + const ids = await interner.resolve(["akash1a", "akash1b"]); + + expect(insertedRows).toEqual([{ table: Accounts, rows: [{ address: "akash1b" }] }]); + expect(ids).toEqual( + new Map([ + ["akash1a", 1], + ["akash1b", 2] + ]) + ); + }); + + it("re-selects addresses lost to a concurrent insert", async () => { + const { interner } = setup({ selectResults: [[], [{ id: 5, address: "akash1c" }]], insertReturning: [] }); + + const ids = await interner.resolve(["akash1c"]); + + expect(ids).toEqual(new Map([["akash1c", 5]])); + }); + + it("dedups repeated addresses so each is interned once", async () => { + const { interner, insertedRows } = setup({ selectResults: [[]], insertReturning: [{ id: 1, address: "akash1a" }] }); + + await interner.resolve(["akash1a", "akash1a"]); + + expect(insertedRows).toEqual([{ table: Accounts, rows: [{ address: "akash1a" }] }]); + }); + + it("chunks the existence lookup so a batch past the bind-parameter limit stays within it", async () => { + const addresses = Array.from({ length: INSERT_CHUNK_SIZE + 1 }, (_, index) => `akash1_${index}`); + const firstChunk = addresses.slice(0, INSERT_CHUNK_SIZE).map((address, index) => ({ id: index + 1, address })); + const secondChunk = addresses.slice(INSERT_CHUNK_SIZE).map((address, index) => ({ id: INSERT_CHUNK_SIZE + 1 + index, address })); + const { interner, insertedRows, selectCount } = setup({ selectResults: [firstChunk, secondChunk] }); + + const ids = await interner.resolve(addresses); + + expect(selectCount()).toBe(2); + expect(insertedRows).toEqual([]); + expect(ids.size).toBe(addresses.length); + }); + + it("does nothing for an empty address set", async () => { + const { interner, insertedRows, selectCount } = setup(); + + const ids = await interner.resolve([]); + + expect(ids.size).toBe(0); + expect(insertedRows).toEqual([]); + expect(selectCount()).toBe(0); + }); + + function setup(input?: { selectResults?: Array>; insertReturning?: Array<{ id: number; address: string }> }) { + const selectResults = [...(input?.selectResults ?? [[]])]; + const insertedRows: Array<{ table: unknown; rows: unknown }> = []; + let selects = 0; + + const dbFake = { + select: () => ({ + from: () => ({ + where: () => { + selects++; + return Promise.resolve(selectResults.shift() ?? []); + } + }) + }), + insert: (table: unknown) => ({ + values: (rows: unknown) => { + insertedRows.push({ table, rows }); + return { onConflictDoNothing: () => ({ returning: () => Promise.resolve(input?.insertReturning ?? []) }) }; + } + }) + }; + + const interner = new AccountInterner(dbFake as unknown as ChainDatabase); + return { interner, insertedRows, selectCount: () => selects }; + } +}); diff --git a/apps/chain-indexer/src/pipeline/balance/account-interner.service.ts b/apps/chain-indexer/src/pipeline/balance/account-interner.service.ts new file mode 100644 index 0000000000..c252d7c5c4 --- /dev/null +++ b/apps/chain-indexer/src/pipeline/balance/account-interner.service.ts @@ -0,0 +1,64 @@ +import { inArray } from "drizzle-orm"; +import chunk from "lodash/chunk"; +import { inject, singleton } from "tsyringe"; + +import { INSERT_CHUNK_SIZE } from "@src/db/insert-chunk-size"; +import { Accounts } from "@src/db/schema"; +import type { ChainDatabase } from "@src/providers/db.provider"; +import { CHAIN_DB } from "@src/providers/db.provider"; + +/** + * Resolves mid-chain addresses to account ids, creating any that don't exist yet. Unlike the genesis + * account-seeder (which owns an empty table and reads ids straight back from `returning()`), addresses + * appear unboundedly during sync, so this holds no permanent cache and resolves per batch: select existing, + * insert the rest with `onConflictDoNothing().returning()`, then re-select any lost to a concurrent writer. + * It runs on the base connection, not the commit transaction, so the interned rows are visible when the + * transaction inserts ledger rows that reference them. + */ +@singleton() +export class AccountInterner { + readonly #db: ChainDatabase; + + constructor(@inject(CHAIN_DB) db: ChainDatabase) { + this.#db = db; + } + + async resolve(addresses: Iterable): Promise> { + const unique = [...new Set(addresses)]; + const idByAddress = new Map(); + + if (unique.length === 0) { + return idByAddress; + } + + await this.#selectInto(idByAddress, unique); + + const missing = unique.filter(address => !idByAddress.has(address)); + if (missing.length === 0) { + return idByAddress; + } + + for (const addressChunk of chunk(missing, INSERT_CHUNK_SIZE)) { + const inserted = await this.#db + .insert(Accounts) + .values(addressChunk.map(address => ({ address }))) + .onConflictDoNothing() + .returning({ id: Accounts.id, address: Accounts.address }); + inserted.forEach(row => idByAddress.set(row.address, row.id)); + } + + const stillMissing = missing.filter(address => !idByAddress.has(address)); + if (stillMissing.length > 0) { + await this.#selectInto(idByAddress, stillMissing); + } + + return idByAddress; + } + + async #selectInto(idByAddress: Map, addresses: string[]): Promise { + for (const addressChunk of chunk(addresses, INSERT_CHUNK_SIZE)) { + const rows = await this.#db.select({ id: Accounts.id, address: Accounts.address }).from(Accounts).where(inArray(Accounts.address, addressChunk)); + rows.forEach(row => idByAddress.set(row.address, row.id)); + } + } +} diff --git a/apps/chain-indexer/src/pipeline/balance/account-tx-deriver.spec.ts b/apps/chain-indexer/src/pipeline/balance/account-tx-deriver.spec.ts new file mode 100644 index 0000000000..130e42cecf --- /dev/null +++ b/apps/chain-indexer/src/pipeline/balance/account-tx-deriver.spec.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from "vitest"; + +import { deriveAccountTxs } from "@src/pipeline/balance/account-tx-deriver"; +import type { DecodedBlock, DecodedEvent, DecodedTransaction } from "@src/pipeline/decoded-block"; + +describe("deriveAccountTxs", () => { + it("records each signer of a transaction with the signer role", () => { + const block = buildBlock([buildTx({ index: 0, signerAddresses: ["akash1signer1", "akash1signer2"] })]); + + expect(deriveAccountTxs(block)).toEqual([ + { address: "akash1signer1", height: 10, txIndex: 0, role: "signer" }, + { address: "akash1signer2", height: 10, txIndex: 0, role: "signer" } + ]); + }); + + it("records the sender and recipient of a transfer event with their roles", () => { + const block = buildBlock([buildTx({ index: 0, events: [event("transfer", { sender: "akash1a", recipient: "akash1b", amount: "1uakt" })] })]); + + expect(deriveAccountTxs(block)).toEqual([ + { address: "akash1a", height: 10, txIndex: 0, role: "sender" }, + { address: "akash1b", height: 10, txIndex: 0, role: "receiver" } + ]); + }); + + it("dedups a repeated (address, tx, role) so the primary key never conflicts within a block", () => { + const block = buildBlock([ + buildTx({ + index: 0, + signerAddresses: ["akash1a", "akash1a"], + events: [ + event("transfer", { sender: "akash1a", recipient: "akash1b", amount: "1uakt" }), + event("transfer", { sender: "akash1a", recipient: "akash1b", amount: "2uakt" }) + ] + }) + ]); + + expect(deriveAccountTxs(block)).toEqual([ + { address: "akash1a", height: 10, txIndex: 0, role: "signer" }, + { address: "akash1a", height: 10, txIndex: 0, role: "sender" }, + { address: "akash1b", height: 10, txIndex: 0, role: "receiver" } + ]); + }); + + it("keeps the same address distinct across roles and transactions", () => { + const block = buildBlock([ + buildTx({ index: 0, signerAddresses: ["akash1a"] }), + buildTx({ index: 1, events: [event("transfer", { sender: "akash1a", recipient: "akash1b", amount: "1uakt" })] }) + ]); + + expect(deriveAccountTxs(block)).toEqual([ + { address: "akash1a", height: 10, txIndex: 0, role: "signer" }, + { address: "akash1a", height: 10, txIndex: 1, role: "sender" }, + { address: "akash1b", height: 10, txIndex: 1, role: "receiver" } + ]); + }); + + it("ignores block-level transfer events, which have no transaction to attribute", () => { + const block: DecodedBlock = { ...buildBlock([]), blockEvents: [event("transfer", { sender: "akash1a", recipient: "akash1b", amount: "1uakt" })] }; + + expect(deriveAccountTxs(block)).toEqual([]); + }); + + function buildBlock(transactions: DecodedTransaction[]): DecodedBlock { + return { + height: 10, + datetime: new Date("2026-08-11T00:00:00Z"), + hash: Buffer.alloc(0), + parentHash: null, + proposerAddress: "PROPOSER", + transactions, + blockEvents: [] + }; + } + + function buildTx(input: { index: number; signerAddresses?: string[]; events?: DecodedEvent[] }): DecodedTransaction { + return { + index: input.index, + hash: Buffer.alloc(0), + code: 0, + gasUsed: 0, + gasWanted: 0, + fee: [], + messages: [], + events: input.events ?? [], + signerAddresses: input.signerAddresses ?? [] + }; + } + + function event(type: string, attributes: Record): DecodedEvent { + return { type, attributes }; + } +}); diff --git a/apps/chain-indexer/src/pipeline/balance/account-tx-deriver.ts b/apps/chain-indexer/src/pipeline/balance/account-tx-deriver.ts new file mode 100644 index 0000000000..0acd8fad3f --- /dev/null +++ b/apps/chain-indexer/src/pipeline/balance/account-tx-deriver.ts @@ -0,0 +1,49 @@ +import type { accountTxRole } from "@src/db/schema"; +import type { DecodedBlock } from "@src/pipeline/decoded-block"; + +export type AccountTxRole = (typeof accountTxRole.enumValues)[number]; + +/** One address's participation in a transaction, before its address is interned to an account id. */ +export interface DerivedAccountTx { + address: string; + height: number; + txIndex: number; + role: AccountTxRole; +} + +/** + * Builds the address activity log for a block: every transaction's signers plus the sender and recipient + * of each of its `transfer` events. Rows are deduped per `(address, txIndex, role)` so they never collide + * on the `account_txs` primary key. Block-level events have no owning transaction and are skipped. + */ +export function deriveAccountTxs(block: DecodedBlock): DerivedAccountTx[] { + const rows: DerivedAccountTx[] = []; + const seen = new Set(); + + const add = (address: string, txIndex: number, role: AccountTxRole) => { + if (!address) { + return; + } + const key = `${address}|${txIndex}|${role}`; + if (seen.has(key)) { + return; + } + seen.add(key); + rows.push({ address, height: block.height, txIndex, role }); + }; + + for (const tx of block.transactions) { + for (const signer of tx.signerAddresses) { + add(signer, tx.index, "signer"); + } + + for (const event of tx.events) { + if (event.type === "transfer") { + add(event.attributes.sender, tx.index, "sender"); + add(event.attributes.recipient, tx.index, "receiver"); + } + } + } + + return rows; +} diff --git a/apps/chain-indexer/src/pipeline/balance/balance-deriver.spec.ts b/apps/chain-indexer/src/pipeline/balance/balance-deriver.spec.ts new file mode 100644 index 0000000000..a89afa42d2 --- /dev/null +++ b/apps/chain-indexer/src/pipeline/balance/balance-deriver.spec.ts @@ -0,0 +1,230 @@ +import { describe, expect, it } from "vitest"; + +import { AKASH_ADDRESS_PREFIX } from "@src/genesis/genesis-address"; +import { deriveBalanceChanges } from "@src/pipeline/balance/balance-deriver"; +import { buildModuleAddressRegistry, deriveModuleAddress } from "@src/pipeline/balance/module-address-registry"; +import type { DecodedBlock, DecodedEvent, DecodedTransaction } from "@src/pipeline/decoded-block"; + +const registry = buildModuleAddressRegistry(AKASH_ADDRESS_PREFIX); +const feeCollector = deriveModuleAddress("fee_collector", AKASH_ADDRESS_PREFIX); + +describe("deriveBalanceChanges", () => { + it("emits a debit and a credit for a simple transfer with correlated counterparties", () => { + const block = buildBlock({ + transactions: [ + buildTx({ + index: 0, + events: [ + event("coin_spent", { spender: "akash1a", amount: "100uakt", msg_index: "0" }, 0), + event("coin_received", { receiver: "akash1b", amount: "100uakt", msg_index: "0" }, 0), + event("transfer", { sender: "akash1a", recipient: "akash1b", amount: "100uakt", msg_index: "0" }, 0) + ] + }) + ] + }); + + expect(deriveBalanceChanges(block, registry)).toEqual([ + { address: "akash1a", counterpartyAddress: "akash1b", denom: "uakt", delta: -100n, reason: "transfer", height: 10, txIndex: 0, eventIndex: 0 }, + { address: "akash1b", counterpartyAddress: "akash1a", denom: "uakt", delta: 100n, reason: "transfer", height: 10, txIndex: 0, eventIndex: 1 } + ]); + }); + + it("assigns a deterministic block-wide event index across txs then block events, expanding per denom in amount order", () => { + const block = buildBlock({ + transactions: [ + buildTx({ index: 0, events: [event("coin_spent", { spender: "akash1a", amount: "5uakt,3uatom" }, undefined)] }), + buildTx({ index: 1, events: [event("coin_received", { receiver: "akash1b", amount: "7uakt" }, undefined)] }) + ], + blockEvents: [event("coin_received", { receiver: "akash1c", amount: "9uakt" }, undefined)] + }); + + expect( + deriveBalanceChanges(block, registry).map(change => ({ + eventIndex: change.eventIndex, + address: change.address, + denom: change.denom, + delta: change.delta, + txIndex: change.txIndex + })) + ).toEqual([ + { eventIndex: 0, address: "akash1a", denom: "uakt", delta: -5n, txIndex: 0 }, + { eventIndex: 1, address: "akash1a", denom: "uatom", delta: -3n, txIndex: 0 }, + { eventIndex: 2, address: "akash1b", denom: "uakt", delta: 7n, txIndex: 1 }, + { eventIndex: 3, address: "akash1c", denom: "uakt", delta: 9n, txIndex: null } + ]); + }); + + it("classifies a fee payment to the fee collector", () => { + const block = buildBlock({ + transactions: [ + buildTx({ + index: 0, + events: [ + event("coin_spent", { spender: "akash1payer", amount: "500uakt" }, undefined), + event("coin_received", { receiver: feeCollector, amount: "500uakt" }, undefined), + event("transfer", { sender: "akash1payer", recipient: feeCollector, amount: "500uakt" }, undefined) + ] + }) + ] + }); + + const changes = deriveBalanceChanges(block, registry); + expect(changes.map(change => change.reason)).toEqual(["fee", "fee"]); + expect(changes[0]).toMatchObject({ address: "akash1payer", counterpartyAddress: feeCollector, reason: "fee" }); + }); + + it("classifies a block-level inflation mint from the coincident coinbase event", () => { + const mintModule = deriveModuleAddress("mint", AKASH_ADDRESS_PREFIX); + const block = buildBlock({ + transactions: [], + blockEvents: [ + event("coinbase", { minter: mintModule, amount: "1000uakt" }, undefined), + event("coin_received", { receiver: mintModule, amount: "1000uakt" }, undefined) + ] + }); + + const changes = deriveBalanceChanges(block, registry); + expect(changes).toEqual([ + { address: mintModule, counterpartyAddress: null, denom: "uakt", delta: 1000n, reason: "mint", height: 10, txIndex: null, eventIndex: 0 } + ]); + }); + + it("classifies a debit coincident with a burn event as a burn", () => { + const block = buildBlock({ + transactions: [ + buildTx({ + index: 0, + events: [ + event("coin_spent", { spender: "akash1burner", amount: "42uakt" }, undefined), + event("burn", { burner: "akash1burner", amount: "42uakt" }, undefined) + ] + }) + ] + }); + + expect(deriveBalanceChanges(block, registry)[0]).toMatchObject({ reason: "burn", delta: -42n }); + }); + + it("applies the slash reason only to the coincident burn leg, leaving the block's inflation mint a mint", () => { + const mintModule = deriveModuleAddress("mint", AKASH_ADDRESS_PREFIX); + const bondedPool = deriveModuleAddress("bonded_tokens_pool", AKASH_ADDRESS_PREFIX); + const block = buildBlock({ + transactions: [], + blockEvents: [ + event("coinbase", { minter: mintModule, amount: "1000uakt" }, undefined), + event("coin_received", { receiver: mintModule, amount: "1000uakt" }, undefined), + event("slash", { address: "akashvalcons1jailed", amount: "50uakt" }, undefined), + event("coin_spent", { spender: bondedPool, amount: "50uakt" }, undefined), + event("burn", { burner: bondedPool, amount: "50uakt" }, undefined) + ] + }); + + const byAddress = new Map(deriveBalanceChanges(block, registry).map(change => [change.address, change.reason])); + expect(byAddress.get(mintModule)).toBe("mint"); + expect(byAddress.get(bondedPool)).toBe("slash"); + }); + + it("leaves an unrelated module burn a burn when it shares a block scope with a validator slash", () => { + const bondedPool = deriveModuleAddress("bonded_tokens_pool", AKASH_ADDRESS_PREFIX); + const govModule = deriveModuleAddress("gov", AKASH_ADDRESS_PREFIX); + const block = buildBlock({ + transactions: [], + blockEvents: [ + event("slash", { address: "akashvalcons1jailed", amount: "50uakt" }, undefined), + event("coin_spent", { spender: bondedPool, amount: "50uakt" }, undefined), + event("burn", { burner: bondedPool, amount: "50uakt" }, undefined), + event("coin_spent", { spender: govModule, amount: "10uakt" }, undefined), + event("burn", { burner: govModule, amount: "10uakt" }, undefined) + ] + }); + + const byAddress = new Map(deriveBalanceChanges(block, registry).map(change => [change.address, change.reason])); + expect(byAddress.get(bondedPool)).toBe("slash"); + expect(byAddress.get(govModule)).toBe("burn"); + }); + + it("correlates each debit to the transfer of matching amount when one sender pays several recipients in a block", () => { + const escrow = deriveModuleAddress("escrow", AKASH_ADDRESS_PREFIX); + const block = buildBlock({ + transactions: [], + blockEvents: [ + event("transfer", { sender: escrow, recipient: "akash1providerA", amount: "100uakt" }, undefined), + event("transfer", { sender: escrow, recipient: "akash1providerB", amount: "50uakt" }, undefined), + event("coin_spent", { spender: escrow, amount: "100uakt" }, undefined), + event("coin_spent", { spender: escrow, amount: "50uakt" }, undefined), + event("coin_received", { receiver: "akash1providerA", amount: "100uakt" }, undefined), + event("coin_received", { receiver: "akash1providerB", amount: "50uakt" }, undefined) + ] + }); + + const debits = deriveBalanceChanges(block, registry).filter(change => change.address === escrow); + expect(debits.find(change => change.delta === -100n)?.counterpartyAddress).toBe("akash1providerA"); + expect(debits.find(change => change.delta === -50n)?.counterpartyAddress).toBe("akash1providerB"); + }); + + it("classifies a distribution reward withdrawal using the message type at the coin's msg index", () => { + const distribution = deriveModuleAddress("distribution", AKASH_ADDRESS_PREFIX); + const block = buildBlock({ + transactions: [ + buildTx({ + index: 0, + messages: [{ index: 0, typeUrl: "/cosmos.distribution.v1beta1.MsgWithdrawValidatorCommission", body: null }], + events: [ + event("coin_spent", { spender: distribution, amount: "8uakt", msg_index: "0" }, 0), + event("coin_received", { receiver: "akash1val", amount: "8uakt", msg_index: "0" }, 0), + event("transfer", { sender: distribution, recipient: "akash1val", amount: "8uakt", msg_index: "0" }, 0) + ] + }) + ] + }); + + const creditToValidator = deriveBalanceChanges(block, registry).find(change => change.address === "akash1val"); + expect(creditToValidator?.reason).toBe("commission"); + }); + + it("ignores transfer, coinbase and message events as delta sources", () => { + const block = buildBlock({ + transactions: [ + buildTx({ + index: 0, + events: [ + event("transfer", { sender: "akash1a", recipient: "akash1b", amount: "1uakt" }, undefined), + event("message", { action: "/cosmos.bank.v1beta1.MsgSend" }, undefined) + ] + }) + ] + }); + + expect(deriveBalanceChanges(block, registry)).toEqual([]); + }); + + function buildBlock(input: { transactions: DecodedTransaction[]; blockEvents?: DecodedEvent[] }): DecodedBlock { + return { + height: 10, + datetime: new Date("2026-08-11T00:00:00Z"), + hash: Buffer.alloc(0), + parentHash: null, + proposerAddress: "PROPOSER", + transactions: input.transactions, + blockEvents: input.blockEvents ?? [] + }; + } + + function buildTx(input: { index: number; events: DecodedEvent[]; messages?: DecodedTransaction["messages"] }): DecodedTransaction { + return { + index: input.index, + hash: Buffer.alloc(0), + code: 0, + gasUsed: 0, + gasWanted: 0, + fee: [], + messages: input.messages ?? [], + events: input.events, + signerAddresses: [] + }; + } + + function event(type: string, attributes: Record, msgIndex: number | undefined): DecodedEvent { + return msgIndex === undefined ? { type, attributes } : { type, attributes, msgIndex }; + } +}); diff --git a/apps/chain-indexer/src/pipeline/balance/balance-deriver.ts b/apps/chain-indexer/src/pipeline/balance/balance-deriver.ts new file mode 100644 index 0000000000..9ebca7c13b --- /dev/null +++ b/apps/chain-indexer/src/pipeline/balance/balance-deriver.ts @@ -0,0 +1,154 @@ +import { parseCoins } from "@src/pipeline/balance/coin-amount"; +import type { ModuleAddressRegistry, ModuleRole } from "@src/pipeline/balance/module-address-registry"; +import type { BalanceReason } from "@src/pipeline/balance/reason-classifier"; +import { classifyReason } from "@src/pipeline/balance/reason-classifier"; +import type { DecodedBlock, DecodedEvent, DecodedTransaction } from "@src/pipeline/decoded-block"; + +/** A single balance movement before its address is interned to an account id. Deltas come only from coin_spent/coin_received. */ +export interface DerivedBalanceChange { + address: string; + counterpartyAddress: string | null; + denom: string; + delta: bigint; + reason: BalanceReason; + height: number; + txIndex: number | null; + eventIndex: number; +} + +const BLOCK_SCOPE = "block"; + +/** A slash burns from the staking pools, so a slashing block's burn leg only counts as the slash when its holder is one of those pools. */ +const STAKING_POOL_ROLES: ReadonlySet = new Set(["bonded_tokens_pool", "not_bonded_tokens_pool"]); + +interface ParsedTransfer { + sender: string; + recipient: string; + coins: Map; +} + +/** + * Per-scope classification context: the transfers to correlate a counterparty against, plus which addresses + * minted/burned and whether a slash occurred. `slashed` is scope-wide because a `slash` event names the + * validator, not the staking pool whose coins actually move, so it is only trusted to reclassify the + * coincident burn leg (see `deriveBalanceChanges`), never every movement sharing the scope. + */ +interface ScopeContext { + transfers: ParsedTransfer[]; + minters: Set; + burners: Set; + slashed: boolean; +} + +interface EventSource { + events: DecodedEvent[]; + txIndex: number | null; + msgTypeByIndex: Map; +} + +function scopeKeyOf(event: DecodedEvent): number | string { + return event.msgIndex ?? BLOCK_SCOPE; +} + +function buildScopeContexts(events: DecodedEvent[]): Map { + const scopes = new Map(); + const scopeOf = (event: DecodedEvent) => { + const key = scopeKeyOf(event); + const existing = scopes.get(key); + if (existing) { + return existing; + } + const created: ScopeContext = { transfers: [], minters: new Set(), burners: new Set(), slashed: false }; + scopes.set(key, created); + return created; + }; + + for (const event of events) { + const scope = scopeOf(event); + if (event.type === "transfer") { + scope.transfers.push({ + sender: event.attributes.sender, + recipient: event.attributes.recipient, + coins: new Map(parseCoins(event.attributes.amount ?? "").map(coin => [coin.denom, coin.amount])) + }); + } else if (event.type === "coinbase" && event.attributes.minter) { + scope.minters.add(event.attributes.minter); + } else if (event.type === "burn" && event.attributes.burner) { + scope.burners.add(event.attributes.burner); + } else if (event.type === "slash") { + scope.slashed = true; + } + } + + return scopes; +} + +function correlateCounterparty(scope: ScopeContext, holder: string, denom: string, amount: bigint, direction: "spent" | "received"): string | null { + const matchesHolder = (transfer: ParsedTransfer) => (direction === "spent" ? transfer.sender === holder : transfer.recipient === holder); + const other = (transfer: ParsedTransfer) => (direction === "spent" ? transfer.recipient : transfer.sender); + + const byAmount = scope.transfers.find(transfer => matchesHolder(transfer) && transfer.coins.get(denom) === amount); + const byDenom = scope.transfers.find(transfer => matchesHolder(transfer) && transfer.coins.has(denom)); + const byHolder = scope.transfers.find(matchesHolder); + const match = byAmount ?? byDenom ?? byHolder; + + return match ? other(match) : null; +} + +/** + * Turns a block's coin events into ordered balance movements. Deltas come solely from `coin_spent` + * (holder −amount) and `coin_received` (holder +amount); `transfer`/`coinbase`/`burn`/`slash` only + * inform the counterparty and reason. The `event_index` is a deterministic block-wide sequence — each tx + * in ascending order, its coin events in array order expanded per denom, then block-level events — so a + * re-derivation of the same block reproduces the exact `(height, event_index)` idempotency keys. + */ +export function deriveBalanceChanges(block: DecodedBlock, registry: ModuleAddressRegistry): DerivedBalanceChange[] { + const sources: EventSource[] = [ + ...[...block.transactions].sort((a, b) => a.index - b.index).map(tx => ({ events: tx.events, txIndex: tx.index, msgTypeByIndex: msgTypeByIndexOf(tx) })), + { events: block.blockEvents, txIndex: null, msgTypeByIndex: new Map() } + ]; + + const changes: DerivedBalanceChange[] = []; + let eventIndex = 0; + + for (const source of sources) { + const scopes = buildScopeContexts(source.events); + + for (const event of source.events) { + const direction = event.type === "coin_spent" ? "spent" : event.type === "coin_received" ? "received" : null; + if (!direction) { + continue; + } + + const holder = direction === "spent" ? event.attributes.spender : event.attributes.receiver; + const scope = scopes.get(scopeKeyOf(event)) ?? { transfers: [], minters: new Set(), burners: new Set(), slashed: false }; + const holderRole = registry.roleOf(holder); + const msgTypeUrl = event.msgIndex === undefined ? null : source.msgTypeByIndex.get(event.msgIndex) ?? null; + const isMint = direction === "received" && scope.minters.has(holder); + const isBurn = direction === "spent" && scope.burners.has(holder); + const isSlash = scope.slashed && isBurn && holderRole !== undefined && STAKING_POOL_ROLES.has(holderRole); + + for (const coin of parseCoins(event.attributes.amount ?? "")) { + const counterpartyAddress = correlateCounterparty(scope, holder, coin.denom, coin.amount, direction); + const reason = classifyReason({ address: holder, counterpartyAddress, denom: coin.denom, isMint, isBurn, isSlash, msgTypeUrl }, registry); + + changes.push({ + address: holder, + counterpartyAddress, + denom: coin.denom, + delta: direction === "spent" ? -coin.amount : coin.amount, + reason, + height: block.height, + txIndex: source.txIndex, + eventIndex: eventIndex++ + }); + } + } + } + + return changes; +} + +function msgTypeByIndexOf(tx: DecodedTransaction): Map { + return new Map(tx.messages.map(message => [message.index, message.typeUrl])); +} diff --git a/apps/chain-indexer/src/pipeline/balance/balance-writer.service.spec.ts b/apps/chain-indexer/src/pipeline/balance/balance-writer.service.spec.ts new file mode 100644 index 0000000000..3148fefc4f --- /dev/null +++ b/apps/chain-indexer/src/pipeline/balance/balance-writer.service.spec.ts @@ -0,0 +1,171 @@ +import type { SQL } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { describe, expect, it } from "vitest"; + +import { AccountBalances, BalanceChanges } from "@src/db/schema"; +import type { ResolvedBalanceChange } from "@src/pipeline/balance/balance-writer.service"; +import { BalanceWriter } from "@src/pipeline/balance/balance-writer.service"; +import type { ChainTransaction } from "@src/providers/db.provider"; + +describe(BalanceWriter.name, () => { + it("computes each ledger entry's running balance from the ledger baseline", async () => { + const { writer, tx, balanceChangeRows } = setup({ baseline: [{ accountId: 1, denom: "uakt", balanceAfter: "100" }] }); + + await writer.write(tx, [change({ accountId: 1, delta: 50n, eventIndex: 0 }), change({ accountId: 1, delta: -30n, eventIndex: 1 })]); + + expect(balanceChangeRows().map(row => row.balanceAfter)).toEqual(["150", "120"]); + }); + + it("seeds the running balance from zero for an account with no prior ledger history", async () => { + const { writer, tx, balanceChangeRows } = setup(); + + await writer.write(tx, [change({ accountId: 2, delta: 10n, eventIndex: 0 })]); + + expect(balanceChangeRows().map(row => row.balanceAfter)).toEqual(["10"]); + }); + + it("carries the running balance across non-adjacent heights within a batch", async () => { + const { writer, tx, balanceChangeRows } = setup(); + + await writer.write(tx, [change({ accountId: 1, delta: 5n, height: 10, eventIndex: 0 }), change({ accountId: 1, delta: -2n, height: 13, eventIndex: 0 })]); + + expect(balanceChangeRows().map(row => row.balanceAfter)).toEqual(["5", "3"]); + }); + + it("sorts intents by height then event index before accumulating", async () => { + const { writer, tx, balanceChangeRows } = setup(); + + await writer.write(tx, [change({ accountId: 1, delta: -2n, height: 13, eventIndex: 0 }), change({ accountId: 1, delta: 5n, height: 10, eventIndex: 0 })]); + + expect(balanceChangeRows().map(row => ({ height: row.height, balanceAfter: row.balanceAfter }))).toEqual([ + { height: 10, balanceAfter: "5" }, + { height: 13, balanceAfter: "3" } + ]); + }); + + it("advances the current-balance snapshot additively with the summed net delta of the inserted rows", async () => { + const { writer, tx, balanceUpserts, conflictSet } = setup(); + + await writer.write(tx, [change({ accountId: 1, delta: 50n, eventIndex: 0 }), change({ accountId: 1, delta: -30n, eventIndex: 1 })]); + + expect(balanceUpserts()).toEqual([{ accountId: 1, denom: "uakt", amount: "20" }]); + expect(new PgDialect().sqlToQuery(conflictSet()!.amount).sql).toBe('"cosmos"."account_balances"."amount" + EXCLUDED.amount'); + }); + + it("applies zero deltas when a re-commit inserts no new rows", async () => { + const { writer, tx, balanceUpserts } = setup({ insertReturning: [] }); + + await writer.write(tx, [change({ accountId: 1, delta: 50n, eventIndex: 0 })]); + + expect(balanceUpserts()).toEqual([]); + }); + + it("keeps running balances correct while advancing the snapshot only by newly-inserted rows when a batch straddles the sync frontier", async () => { + const { writer, tx, balanceChangeRows, balanceUpserts } = setup({ + baseline: [{ accountId: 1, denom: "uakt", balanceAfter: "100" }], + insertReturning: [{ accountId: 1, denom: "uakt", delta: "7" }] + }); + + await writer.write(tx, [change({ accountId: 1, delta: 5n, height: 10, eventIndex: 0 }), change({ accountId: 1, delta: 7n, height: 11, eventIndex: 0 })]); + + expect(balanceChangeRows().map(row => row.balanceAfter)).toEqual(["105", "112"]); + expect(balanceUpserts()).toEqual([{ accountId: 1, denom: "uakt", amount: "7" }]); + }); + + it("does nothing for an empty intent set", async () => { + const { writer, tx, calls } = setup(); + + await writer.write(tx, []); + + expect(calls()).toBe(0); + }); + + it("reads the baseline in chunks so a batch touching more accounts than the chunk size stays under the bind-parameter limit", async () => { + const chunkSize = 2000; + const accountIds = Array.from({ length: chunkSize + 1 }, (_, index) => index + 1); + const { writer, tx, balanceChangeRows, baselineSelects } = setup({ + baselineByChunk: [[{ accountId: 1, denom: "uakt", balanceAfter: "100" }], [{ accountId: chunkSize + 1, denom: "uakt", balanceAfter: "500" }]] + }); + + await writer.write( + tx, + accountIds.map((accountId, index) => change({ accountId, delta: 10n, eventIndex: index })) + ); + + expect(baselineSelects()).toBe(2); + const balanceAfterByAccount = new Map(balanceChangeRows().map(row => [row.accountId, row.balanceAfter])); + expect(balanceAfterByAccount.get(1)).toBe("110"); + expect(balanceAfterByAccount.get(chunkSize + 1)).toBe("510"); + }); + + function change(input: Partial): ResolvedBalanceChange { + return { + accountId: 1, + counterpartyAccountId: null, + denom: "uakt", + delta: 0n, + reason: "transfer", + height: 10, + txIndex: 0, + eventIndex: 0, + ...input + }; + } + + function setup(input?: { + baseline?: Array<{ accountId: number; denom: string; balanceAfter: string }>; + baselineByChunk?: Array>; + insertReturning?: Array<{ accountId: number; denom: string; delta: string }>; + }) { + const balanceChangeInserts: Record[] = []; + const balanceBalanceUpserts: Record[] = []; + const baselineByChunk = [...(input?.baselineByChunk ?? [])]; + let conflictSet: { amount: SQL } | undefined; + let calls = 0; + let baselineSelects = 0; + + const txFake = { + selectDistinctOn: () => ({ + from: () => ({ + where: () => ({ + orderBy: () => { + calls++; + baselineSelects++; + return Promise.resolve(baselineByChunk.length > 0 ? baselineByChunk.shift()! : input?.baseline ?? []); + } + }) + }) + }), + insert: (table: unknown) => ({ + values: (rows: Record[]) => { + calls++; + if (table === BalanceChanges) { + balanceChangeInserts.push(...rows); + } else if (table === AccountBalances) { + balanceBalanceUpserts.push(...rows); + } + return { + onConflictDoNothing: () => ({ + returning: () => Promise.resolve(input?.insertReturning ?? rows.map(row => ({ accountId: row.accountId, denom: row.denom, delta: row.delta }))) + }), + onConflictDoUpdate: (config: { set: { amount: SQL } }) => { + conflictSet = config.set; + return Promise.resolve(); + } + }; + } + }) + }; + + const writer = new BalanceWriter(); + return { + writer, + tx: txFake as unknown as ChainTransaction, + balanceChangeRows: () => balanceChangeInserts, + balanceUpserts: () => balanceBalanceUpserts, + conflictSet: () => conflictSet, + calls: () => calls, + baselineSelects: () => baselineSelects + }; + } +}); diff --git a/apps/chain-indexer/src/pipeline/balance/balance-writer.service.ts b/apps/chain-indexer/src/pipeline/balance/balance-writer.service.ts new file mode 100644 index 0000000000..162fea71e9 --- /dev/null +++ b/apps/chain-indexer/src/pipeline/balance/balance-writer.service.ts @@ -0,0 +1,152 @@ +import { and, desc, inArray, lt, sql } from "drizzle-orm"; +import chunk from "lodash/chunk"; +import { singleton } from "tsyringe"; + +import { INSERT_CHUNK_SIZE } from "@src/db/insert-chunk-size"; +import { AccountBalances, BalanceChanges } from "@src/db/schema"; +import type { BalanceReason } from "@src/pipeline/balance/reason-classifier"; +import type { ChainTransaction } from "@src/providers/db.provider"; + +/** A derived balance change whose addresses have been interned to account ids, ready to persist. */ +export interface ResolvedBalanceChange { + accountId: number; + counterpartyAccountId: number | null; + denom: string; + delta: bigint; + reason: BalanceReason; + height: number; + txIndex: number | null; + eventIndex: number; +} + +const keyOf = (accountId: number, denom: string) => `${accountId}:${denom}`; + +/** + * Appends balance changes to the ledger and folds them into the current-balance snapshot, idempotently. + * Runs inside the committer's transaction. The `(height, event_index)` unique index is the serialization + * point: `onConflictDoNothing().returning()` yields only rows this call actually inserted, so re-committing + * a block (rolling deploy, backfill overlapping the frontier) inserts zero rows and applies zero deltas. + * The current-balance snapshot is only ever advanced by the returned rows, so it stays a pure projection of + * the ledger and remains rebuildable from it. + */ +@singleton() +export class BalanceWriter { + async write(tx: ChainTransaction, intents: ResolvedBalanceChange[]): Promise { + if (intents.length === 0) { + return; + } + + const ordered = [...intents].sort((a, b) => a.height - b.height || a.eventIndex - b.eventIndex); + const firstHeight = ordered[0].height; + + const baseline = await this.#readLedgerBaseline(tx, ordered, firstHeight); + const changeRows = this.#accumulateRunningBalances(ordered, baseline); + + const inserted = await this.#insertChanges(tx, changeRows); + if (inserted.length === 0) { + return; + } + + await this.#applyNetDeltas(tx, inserted); + } + + /** + * The running balance seeds from the ledger — the `balance_after` of the last change strictly before this + * batch — not from `account_balances`, whose snapshot a concurrent frontier writer may already have advanced. + */ + async #readLedgerBaseline(tx: ChainTransaction, intents: ResolvedBalanceChange[], firstHeight: number): Promise> { + const accountIds = [...new Set(intents.map(intent => intent.accountId))]; + const denoms = [...new Set(intents.map(intent => intent.denom))]; + const touched = new Set(intents.map(intent => keyOf(intent.accountId, intent.denom))); + + const baseline = new Map(); + for (const accountIdChunk of chunk(accountIds, INSERT_CHUNK_SIZE)) { + const rows = await tx + .selectDistinctOn([BalanceChanges.accountId, BalanceChanges.denom], { + accountId: BalanceChanges.accountId, + denom: BalanceChanges.denom, + balanceAfter: BalanceChanges.balanceAfter + }) + .from(BalanceChanges) + .where(and(lt(BalanceChanges.height, firstHeight), inArray(BalanceChanges.accountId, accountIdChunk), inArray(BalanceChanges.denom, denoms))) + .orderBy(BalanceChanges.accountId, BalanceChanges.denom, desc(BalanceChanges.height), desc(BalanceChanges.eventIndex)); + + for (const row of rows) { + const key = keyOf(row.accountId, row.denom); + if (touched.has(key)) { + baseline.set(key, BigInt(row.balanceAfter)); + } + } + } + return baseline; + } + + #accumulateRunningBalances(intents: ResolvedBalanceChange[], baseline: Map): (typeof BalanceChanges.$inferInsert)[] { + const running = new Map(); + + return intents.map(intent => { + const key = keyOf(intent.accountId, intent.denom); + const previous = running.get(key) ?? baseline.get(key) ?? 0n; + const balanceAfter = previous + intent.delta; + running.set(key, balanceAfter); + + return { + accountId: intent.accountId, + denom: intent.denom, + delta: intent.delta.toString(), + balanceAfter: balanceAfter.toString(), + reason: intent.reason, + height: intent.height, + txIndex: intent.txIndex, + eventIndex: intent.eventIndex, + counterpartyAccountId: intent.counterpartyAccountId + }; + }); + } + + async #insertChanges( + tx: ChainTransaction, + changeRows: (typeof BalanceChanges.$inferInsert)[] + ): Promise<{ accountId: number; denom: string; delta: string }[]> { + const inserted: { accountId: number; denom: string; delta: string }[] = []; + + for (const rowChunk of chunk(changeRows, INSERT_CHUNK_SIZE)) { + const returned = await tx + .insert(BalanceChanges) + .values(rowChunk) + .onConflictDoNothing() + .returning({ accountId: BalanceChanges.accountId, denom: BalanceChanges.denom, delta: BalanceChanges.delta }); + inserted.push(...returned); + } + + return inserted; + } + + /** Advances the current balance only by the rows actually inserted, summed per account+denom, so overlapping writers apply each delta exactly once. */ + async #applyNetDeltas(tx: ChainTransaction, inserted: { accountId: number; denom: string; delta: string }[]): Promise { + const netByKey = new Map(); + for (const row of inserted) { + const key = keyOf(row.accountId, row.denom); + const existing = netByKey.get(key); + if (existing) { + existing.amount += BigInt(row.delta); + } else { + netByKey.set(key, { accountId: row.accountId, denom: row.denom, amount: BigInt(row.delta) }); + } + } + + const balanceRows = [...netByKey.values()] + .sort((a, b) => a.accountId - b.accountId || a.denom.localeCompare(b.denom)) + .map(entry => ({ accountId: entry.accountId, denom: entry.denom, amount: entry.amount.toString() })); + + for (const rowChunk of chunk(balanceRows, INSERT_CHUNK_SIZE)) { + await tx + .insert(AccountBalances) + .values(rowChunk) + .onConflictDoUpdate({ + target: [AccountBalances.accountId, AccountBalances.denom], + set: { amount: sql`${AccountBalances.amount} + EXCLUDED.amount` } + }); + } + } +} diff --git a/apps/chain-indexer/src/pipeline/balance/coin-amount.spec.ts b/apps/chain-indexer/src/pipeline/balance/coin-amount.spec.ts new file mode 100644 index 0000000000..8142790775 --- /dev/null +++ b/apps/chain-indexer/src/pipeline/balance/coin-amount.spec.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; + +import { parseCoins } from "@src/pipeline/balance/coin-amount"; + +describe("parseCoins", () => { + it("parses a single coin", () => { + expect(parseCoins("100uakt")).toEqual([{ denom: "uakt", amount: 100n }]); + }); + + it("parses multiple comma-separated coins preserving order", () => { + expect(parseCoins("100uakt,5uatom")).toEqual([ + { denom: "uakt", amount: 100n }, + { denom: "uatom", amount: 5n } + ]); + }); + + it("parses ibc and factory denoms that contain slashes", () => { + expect(parseCoins("7ibc/27394FB092D2ECCD56123C74F36E4C1F926001CEADA9CA97EA622B25F41E5EB2")).toEqual([ + { denom: "ibc/27394FB092D2ECCD56123C74F36E4C1F926001CEADA9CA97EA622B25F41E5EB2", amount: 7n } + ]); + }); + + it("returns an empty array for an empty or whitespace value", () => { + expect(parseCoins("")).toEqual([]); + expect(parseCoins(" ")).toEqual([]); + }); + + it("skips segments that are not a leading integer followed by a denom", () => { + expect(parseCoins("100uakt,,garbage")).toEqual([{ denom: "uakt", amount: 100n }]); + }); + + it("parses amounts far beyond Number.MAX_SAFE_INTEGER without precision loss", () => { + expect(parseCoins("340282366920938463463374607431768211455uakt")).toEqual([{ denom: "uakt", amount: 340282366920938463463374607431768211455n }]); + }); +}); diff --git a/apps/chain-indexer/src/pipeline/balance/coin-amount.ts b/apps/chain-indexer/src/pipeline/balance/coin-amount.ts new file mode 100644 index 0000000000..bd08334dfb --- /dev/null +++ b/apps/chain-indexer/src/pipeline/balance/coin-amount.ts @@ -0,0 +1,24 @@ +export interface CoinAmount { + denom: string; + amount: bigint; +} + +const COIN_PATTERN = /^(\d+)(.+)$/; + +/** + * Parses a Cosmos coin string such as `"100uakt,5uatom"` into typed amounts. Amounts are `bigint` so + * u-denom values above `Number.MAX_SAFE_INTEGER` keep full precision. Denoms may contain slashes + * (ibc/factory), so the split is on the comma and the amount is the leading integer run only. + */ +export function parseCoins(value: string): CoinAmount[] { + const coins: CoinAmount[] = []; + + for (const segment of value.split(",")) { + const match = segment.trim().match(COIN_PATTERN); + if (match) { + coins.push({ amount: BigInt(match[1]), denom: match[2] }); + } + } + + return coins; +} diff --git a/apps/chain-indexer/src/pipeline/balance/module-address-registry.spec.ts b/apps/chain-indexer/src/pipeline/balance/module-address-registry.spec.ts new file mode 100644 index 0000000000..f6fc484b7c --- /dev/null +++ b/apps/chain-indexer/src/pipeline/balance/module-address-registry.spec.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; + +import { AKASH_ADDRESS_PREFIX } from "@src/genesis/genesis-address"; +import { BME_VAULT_ADDRESS, buildModuleAddressRegistry, deriveModuleAddress } from "@src/pipeline/balance/module-address-registry"; + +describe("deriveModuleAddress", () => { + it("matches the well-known cosmos-hub fee collector address", () => { + expect(deriveModuleAddress("fee_collector", "cosmos")).toBe("cosmos17xpfvakm2amg962yls6f84z3kell8c5lserqta"); + }); + + it("matches the well-known cosmos-hub distribution address", () => { + expect(deriveModuleAddress("distribution", "cosmos")).toBe("cosmos1jv65s3grqf6v6jl3dp4t6c9t9rk99cd88lyufl"); + }); +}); + +describe("buildModuleAddressRegistry", () => { + it("maps the derived module addresses back to their role", () => { + const registry = buildModuleAddressRegistry(AKASH_ADDRESS_PREFIX); + + expect(registry.roleOf(deriveModuleAddress("fee_collector", AKASH_ADDRESS_PREFIX))).toBe("fee_collector"); + expect(registry.roleOf(deriveModuleAddress("bonded_tokens_pool", AKASH_ADDRESS_PREFIX))).toBe("bonded_tokens_pool"); + expect(registry.roleOf(deriveModuleAddress("not_bonded_tokens_pool", AKASH_ADDRESS_PREFIX))).toBe("not_bonded_tokens_pool"); + expect(registry.roleOf(deriveModuleAddress("transfer", AKASH_ADDRESS_PREFIX))).toBe("ibc_transfer"); + expect(registry.roleOf(deriveModuleAddress("escrow", AKASH_ADDRESS_PREFIX))).toBe("escrow"); + }); + + it("maps the BME vault address to the bme role", () => { + const registry = buildModuleAddressRegistry(AKASH_ADDRESS_PREFIX); + + expect(registry.roleOf(BME_VAULT_ADDRESS)).toBe("bme_vault"); + }); + + it("returns undefined for a non-module address", () => { + const registry = buildModuleAddressRegistry(AKASH_ADDRESS_PREFIX); + + expect(registry.roleOf("akash1regularuseraddressxxxxxxxxxxxxxxxxxxx")).toBeUndefined(); + }); +}); diff --git a/apps/chain-indexer/src/pipeline/balance/module-address-registry.ts b/apps/chain-indexer/src/pipeline/balance/module-address-registry.ts new file mode 100644 index 0000000000..8b252330ff --- /dev/null +++ b/apps/chain-indexer/src/pipeline/balance/module-address-registry.ts @@ -0,0 +1,58 @@ +import { toBech32 } from "@cosmjs/encoding"; +import { createHash } from "node:crypto"; + +import { AKASH_ADDRESS_PREFIX } from "@src/genesis/genesis-address"; + +/** A recognized system account whose involvement in a coin movement identifies the movement's reason. */ +export type ModuleRole = + | "fee_collector" + | "distribution" + | "mint" + | "gov" + | "bonded_tokens_pool" + | "not_bonded_tokens_pool" + | "ibc_transfer" + | "bme_vault" + | "escrow"; + +/** The Akash BME vault, funded by escrow settlements and MsgMintACT and drained by burns; not a `x/auth` module account. */ +export const BME_VAULT_ADDRESS = "akash1klpwzlvfnw7j8gtdd0cuu9vaw9ermsmd37sg55"; + +const MODULE_NAME_ROLES: Record = { + fee_collector: "fee_collector", + distribution: "distribution", + mint: "mint", + gov: "gov", + bonded_tokens_pool: "bonded_tokens_pool", + not_bonded_tokens_pool: "not_bonded_tokens_pool", + transfer: "ibc_transfer", + escrow: "escrow" +}; + +/** + * The bech32 address of a Cosmos SDK module account: the first 20 bytes of `sha256(moduleName)`, matching + * `authtypes.NewModuleAddress`. Derivation is deterministic, so the classifier can recognize a module + * account without needing it seeded from genesis. + */ +export function deriveModuleAddress(moduleName: string, prefix: string): string { + const digest = createHash("sha256").update(Buffer.from(moduleName)).digest(); + return toBech32(prefix, digest.subarray(0, 20)); +} + +export interface ModuleAddressRegistry { + roleOf(address: string): ModuleRole | undefined; +} + +/** Precomputes the address→role map for every known system account so reason classification is a single map lookup. */ +export function buildModuleAddressRegistry(prefix: string = AKASH_ADDRESS_PREFIX): ModuleAddressRegistry { + const roleByAddress = new Map(); + + for (const [moduleName, role] of Object.entries(MODULE_NAME_ROLES)) { + roleByAddress.set(deriveModuleAddress(moduleName, prefix), role); + } + roleByAddress.set(BME_VAULT_ADDRESS, "bme_vault"); + + return { + roleOf: address => roleByAddress.get(address) + }; +} diff --git a/apps/chain-indexer/src/pipeline/balance/reason-classifier.spec.ts b/apps/chain-indexer/src/pipeline/balance/reason-classifier.spec.ts new file mode 100644 index 0000000000..39dfc4da60 --- /dev/null +++ b/apps/chain-indexer/src/pipeline/balance/reason-classifier.spec.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from "vitest"; + +import { AKASH_ADDRESS_PREFIX } from "@src/genesis/genesis-address"; +import { BME_VAULT_ADDRESS, buildModuleAddressRegistry, deriveModuleAddress } from "@src/pipeline/balance/module-address-registry"; +import type { ReasonContext } from "@src/pipeline/balance/reason-classifier"; +import { classifyReason } from "@src/pipeline/balance/reason-classifier"; + +const registry = buildModuleAddressRegistry(AKASH_ADDRESS_PREFIX); +const moduleAddress = (name: string) => deriveModuleAddress(name, AKASH_ADDRESS_PREFIX); + +describe("classifyReason", () => { + it("classifies a payment to the fee collector as a fee", () => { + expect(classifyReason(context({ counterpartyAddress: moduleAddress("fee_collector") }), registry)).toBe("fee"); + }); + + it("classifies a distribution outflow as a reward", () => { + expect(classifyReason(context({ counterpartyAddress: moduleAddress("distribution") }), registry)).toBe("reward"); + }); + + it("classifies a distribution outflow withdrawn by a validator commission message as commission", () => { + const ctx = context({ counterpartyAddress: moduleAddress("distribution"), msgTypeUrl: "/cosmos.distribution.v1beta1.MsgWithdrawValidatorCommission" }); + expect(classifyReason(ctx, registry)).toBe("commission"); + }); + + it("classifies a flow with the bonded pool as staking", () => { + expect(classifyReason(context({ counterpartyAddress: moduleAddress("bonded_tokens_pool") }), registry)).toBe("staking"); + }); + + it("classifies a flow with the not-bonded pool as staking", () => { + expect(classifyReason(context({ address: moduleAddress("not_bonded_tokens_pool") }), registry)).toBe("staking"); + }); + + it("classifies a gov flow as gov", () => { + expect(classifyReason(context({ counterpartyAddress: moduleAddress("gov") }), registry)).toBe("gov"); + }); + + it("classifies an ibc transfer module flow as ibc", () => { + expect(classifyReason(context({ counterpartyAddress: moduleAddress("transfer") }), registry)).toBe("ibc"); + }); + + it("classifies a flow of an ibc denom as ibc", () => { + expect(classifyReason(context({ denom: "ibc/ABCDEF", counterpartyAddress: "akash1peer" }), registry)).toBe("ibc"); + }); + + it("classifies a deposit whose counterparty is the escrow module as escrow", () => { + expect(classifyReason(context({ counterpartyAddress: moduleAddress("escrow") }), registry)).toBe("escrow"); + }); + + it("classifies a settlement paid out by the escrow module as escrow", () => { + expect(classifyReason(context({ address: moduleAddress("escrow"), counterpartyAddress: "akash1provider" }), registry)).toBe("escrow"); + }); + + it("classifies a flow with the BME vault as bme", () => { + expect(classifyReason(context({ counterpartyAddress: BME_VAULT_ADDRESS }), registry)).toBe("bme"); + }); + + it("prefers slash over any module role", () => { + expect(classifyReason(context({ counterpartyAddress: moduleAddress("bonded_tokens_pool"), isSlash: true }), registry)).toBe("slash"); + }); + + it("classifies a coinbase-coincident credit as mint", () => { + expect(classifyReason(context({ isMint: true }), registry)).toBe("mint"); + }); + + it("classifies a flow whose counterparty is the mint module as mint", () => { + expect(classifyReason(context({ counterpartyAddress: moduleAddress("mint") }), registry)).toBe("mint"); + }); + + it("classifies a burn-coincident debit as burn", () => { + expect(classifyReason(context({ isBurn: true }), registry)).toBe("burn"); + }); + + it("defaults a plain account-to-account movement to transfer", () => { + expect(classifyReason(context({ counterpartyAddress: "akash1peer" }), registry)).toBe("transfer"); + }); + + it("tags each leg of a fee_collector-to-distribution movement by the holder's own role", () => { + const feeCollector = moduleAddress("fee_collector"); + const distribution = moduleAddress("distribution"); + + expect(classifyReason(context({ address: feeCollector, counterpartyAddress: distribution }), registry)).toBe("fee"); + expect(classifyReason(context({ address: distribution, counterpartyAddress: feeCollector }), registry)).toBe("reward"); + }); + + it("tags the mint module's outgoing forwarding leg as mint rather than the counterparty's fee", () => { + expect(classifyReason(context({ address: moduleAddress("mint"), counterpartyAddress: moduleAddress("fee_collector") }), registry)).toBe("mint"); + }); + + function context(overrides: Partial): ReasonContext { + return { + address: "akash1self", + counterpartyAddress: null, + denom: "uakt", + isMint: false, + isBurn: false, + isSlash: false, + msgTypeUrl: null, + ...overrides + }; + } +}); diff --git a/apps/chain-indexer/src/pipeline/balance/reason-classifier.ts b/apps/chain-indexer/src/pipeline/balance/reason-classifier.ts new file mode 100644 index 0000000000..1e1bd748db --- /dev/null +++ b/apps/chain-indexer/src/pipeline/balance/reason-classifier.ts @@ -0,0 +1,60 @@ +import type { balanceChangeReason } from "@src/db/schema"; +import type { ModuleAddressRegistry } from "@src/pipeline/balance/module-address-registry"; + +export type BalanceReason = (typeof balanceChangeReason.enumValues)[number]; + +/** What the classifier knows about one coin movement: who moved it, the correlated counterparty, and any coincident mint/burn/slash. */ +export interface ReasonContext { + address: string; + counterpartyAddress: string | null; + denom: string; + isMint: boolean; + isBurn: boolean; + isSlash: boolean; + msgTypeUrl: string | null; +} + +const WITHDRAW_VALIDATOR_COMMISSION = "/cosmos.distribution.v1beta1.MsgWithdrawValidatorCommission"; + +/** + * MVP reason heuristic. Coincident mint/burn/slash win first (they are unambiguous), then the holder's own + * module role, falling back to the counterparty's, then the denom. Preferring the holder's own role keeps each + * leg of a module-to-module movement (e.g. fee_collector to distribution every block) tagged by the module + * whose balance actually changed, rather than mirroring the counterparty. Anything unrecognized is a plain + * `transfer`. Escrow-module movements classify as `escrow`; per-deployment/lease attribution of that escrow is + * deliberately left for later. + */ +export function classifyReason(ctx: ReasonContext, registry: ModuleAddressRegistry): BalanceReason { + if (ctx.isSlash) { + return "slash"; + } + if (ctx.isMint) { + return "mint"; + } + if (ctx.isBurn) { + return "burn"; + } + + const role = registry.roleOf(ctx.address) ?? (ctx.counterpartyAddress ? registry.roleOf(ctx.counterpartyAddress) : undefined); + switch (role) { + case "mint": + return "mint"; + case "fee_collector": + return "fee"; + case "distribution": + return ctx.msgTypeUrl === WITHDRAW_VALIDATOR_COMMISSION ? "commission" : "reward"; + case "bonded_tokens_pool": + case "not_bonded_tokens_pool": + return "staking"; + case "gov": + return "gov"; + case "ibc_transfer": + return "ibc"; + case "escrow": + return "escrow"; + case "bme_vault": + return "bme"; + } + + return ctx.denom.startsWith("ibc/") ? "ibc" : "transfer"; +} diff --git a/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts b/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts index 3da89ecba0..6c542d2268 100644 --- a/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts +++ b/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts @@ -1,13 +1,17 @@ import type { SQL } from "drizzle-orm"; import { PgDialect } from "drizzle-orm/pg-core"; import { describe, expect, it } from "vitest"; +import { mock } from "vitest-mock-extended"; -import { Blocks, IndexerState, Messages, MessageTypes } from "@src/db/schema"; +import { AccountTxs, Blocks, IndexerState, Messages, MessageTypes } from "@src/db/schema"; +import type { AccountInterner } from "@src/pipeline/balance/account-interner.service"; +import type { BalanceWriter } from "@src/pipeline/balance/balance-writer.service"; import { BlockCommitterService } from "@src/pipeline/block-committer.service"; -import type { DecodedBlock } from "@src/pipeline/decoded-block"; +import type { DecodedBlock, DecodedEvent } from "@src/pipeline/decoded-block"; import type { ChainDatabase } from "@src/providers/db.provider"; const MSG_SEND = "/cosmos.bank.v1beta1.MsgSend"; +const BALANCE_WRITE = Symbol("balance_write"); describe(BlockCommitterService.name, () => { it("reuses ids of message types that already exist instead of inserting them", async () => { @@ -99,6 +103,43 @@ describe(BlockCommitterService.name, () => { }); }); + describe("balance ledger and activity log", () => { + it("writes balances then the activity log inside the transaction, after messages and before the checkpoint", async () => { + const { committer, insertedRows } = setup({ selectResults: [[{ id: 7, type: MSG_SEND }]] }); + + await committer.commit(buildBlock([MSG_SEND], 10, { signerAddresses: ["akash1signer"], events: [transfer("akash1signer", "akash1b", "1uakt")] })); + + const order = insertedRows.map(call => call.table); + expect(order.indexOf(Messages)).toBeLessThan(order.indexOf(BALANCE_WRITE)); + expect(order.indexOf(BALANCE_WRITE)).toBeLessThan(order.indexOf(AccountTxs)); + expect(order.indexOf(AccountTxs)).toBeLessThan(order.indexOf(IndexerState)); + }); + + it("passes balance intents with interned account ids to the balance writer", async () => { + const { committer, balanceWriter } = setup({ selectResults: [[{ id: 7, type: MSG_SEND }]] }); + + await committer.commit(buildBlock([MSG_SEND], 10, { events: [coinSpent("akash1a", "100uakt")] })); + + expect(balanceWriter.write.mock.calls[0][1]).toEqual([ + expect.objectContaining({ accountId: 1, counterpartyAccountId: null, denom: "uakt", delta: -100n, height: 10 }) + ]); + }); + + it("interns signers, spenders, receivers and counterparties in one pass", async () => { + const { committer, interner } = setup({ selectResults: [[{ id: 7, type: MSG_SEND }]] }); + + await committer.commit( + buildBlock([MSG_SEND], 10, { + signerAddresses: ["akash1signer"], + events: [coinSpent("akash1a", "1uakt"), coinReceived("akash1b", "1uakt"), transfer("akash1a", "akash1b", "1uakt")] + }) + ); + + const interned = new Set([...interner.resolve.mock.calls[0][0]]); + expect(interned).toEqual(new Set(["akash1a", "akash1b", "akash1signer"])); + }); + }); + function setup(input?: { selectResults?: Array>; insertReturning?: Array<{ id: number; type: string }> }) { const selectResults = [...(input?.selectResults ?? [[]])]; const insertedRows: Array<{ table: unknown; rows: unknown }> = []; @@ -124,11 +165,23 @@ describe(BlockCommitterService.name, () => { transaction: (callback: (tx: unknown) => Promise) => callback(dbFake) }; - const committer = new BlockCommitterService(dbFake as unknown as ChainDatabase); - return { committer, insertedRows, conflictUpdates }; + const interner = mock(); + interner.resolve.mockImplementation(async addresses => new Map([...addresses].map((address, index) => [address, index + 1]))); + + const balanceWriter = mock(); + balanceWriter.write.mockImplementation(async () => { + insertedRows.push({ table: BALANCE_WRITE, rows: [] }); + }); + + const committer = new BlockCommitterService(dbFake as unknown as ChainDatabase, interner, balanceWriter); + return { committer, insertedRows, conflictUpdates, interner, balanceWriter }; } - function buildBlock(typeUrls: string[], height = 10): DecodedBlock { + function buildBlock( + typeUrls: string[], + height = 10, + tx?: { events?: DecodedEvent[]; signerAddresses?: string[]; blockEvents?: DecodedEvent[] } + ): DecodedBlock { return { height, datetime: new Date("2026-08-11T00:00:00Z"), @@ -143,9 +196,24 @@ describe(BlockCommitterService.name, () => { gasUsed: 0, gasWanted: 0, fee: [], - messages: typeUrls.map((typeUrl, index) => ({ index, typeUrl, body: null })) + messages: typeUrls.map((typeUrl, index) => ({ index, typeUrl, body: null })), + events: tx?.events ?? [], + signerAddresses: tx?.signerAddresses ?? [] } - ] + ], + blockEvents: tx?.blockEvents ?? [] }; } + + function coinSpent(spender: string, amount: string): DecodedEvent { + return { type: "coin_spent", attributes: { spender, amount } }; + } + + function coinReceived(receiver: string, amount: string): DecodedEvent { + return { type: "coin_received", attributes: { receiver, amount } }; + } + + function transfer(sender: string, recipient: string, amount: string): DecodedEvent { + return { type: "transfer", attributes: { sender, recipient, amount } }; + } }); diff --git a/apps/chain-indexer/src/pipeline/block-committer.service.ts b/apps/chain-indexer/src/pipeline/block-committer.service.ts index 17935209f2..b1e9a97c51 100644 --- a/apps/chain-indexer/src/pipeline/block-committer.service.ts +++ b/apps/chain-indexer/src/pipeline/block-committer.service.ts @@ -1,9 +1,16 @@ import { inArray, sql } from "drizzle-orm"; -import chunk from "lodash/chunk"; import { inject, singleton } from "tsyringe"; -import { INSERT_CHUNK_SIZE } from "@src/db/insert-chunk-size"; -import { Blocks, IndexerState, Messages, MessageTypes, Transactions } from "@src/db/schema"; +import { insertChunked } from "@src/db/insert-chunked"; +import { AccountTxs, Blocks, IndexerState, Messages, MessageTypes, Transactions } from "@src/db/schema"; +import { AccountInterner } from "@src/pipeline/balance/account-interner.service"; +import type { DerivedAccountTx } from "@src/pipeline/balance/account-tx-deriver"; +import { deriveAccountTxs } from "@src/pipeline/balance/account-tx-deriver"; +import type { DerivedBalanceChange } from "@src/pipeline/balance/balance-deriver"; +import { deriveBalanceChanges } from "@src/pipeline/balance/balance-deriver"; +import type { ResolvedBalanceChange } from "@src/pipeline/balance/balance-writer.service"; +import { BalanceWriter } from "@src/pipeline/balance/balance-writer.service"; +import { buildModuleAddressRegistry } from "@src/pipeline/balance/module-address-registry"; import type { DecodedBlock } from "@src/pipeline/decoded-block"; import type { ChainDatabase } from "@src/providers/db.provider"; import { CHAIN_DB } from "@src/providers/db.provider"; @@ -13,10 +20,15 @@ export const SYNC_STREAM = "sync"; @singleton() export class BlockCommitterService { readonly #db: ChainDatabase; + readonly #interner: AccountInterner; + readonly #balanceWriter: BalanceWriter; + readonly #moduleRegistry = buildModuleAddressRegistry(); readonly #typeIds = new Map(); - constructor(@inject(CHAIN_DB) db: ChainDatabase) { + constructor(@inject(CHAIN_DB) db: ChainDatabase, @inject(AccountInterner) interner: AccountInterner, @inject(BalanceWriter) balanceWriter: BalanceWriter) { this.#db = db; + this.#interner = interner; + this.#balanceWriter = balanceWriter; } async commit(block: DecodedBlock): Promise { @@ -70,20 +82,21 @@ export class BlockCommitterService { ) ); + const balanceChanges = blocks.flatMap(block => deriveBalanceChanges(block, this.#moduleRegistry)); + const accountTxs = blocks.flatMap(block => deriveAccountTxs(block)); + const accountIds = await this.#internAccounts(balanceChanges, accountTxs); + const balanceIntents = this.#resolveBalanceChanges(balanceChanges, accountIds); + const accountTxRows = this.#resolveAccountTxs(accountTxs, accountIds); + const lastHeight = blocks[blocks.length - 1].height; await this.#db.transaction(async tx => { - for (const blockChunk of chunk(blockRows, INSERT_CHUNK_SIZE)) { - await tx.insert(Blocks).values(blockChunk).onConflictDoNothing(); - } - - for (const transactionChunk of chunk(transactionRows, INSERT_CHUNK_SIZE)) { - await tx.insert(Transactions).values(transactionChunk).onConflictDoNothing(); - } + await insertChunked(tx, Blocks, blockRows); + await insertChunked(tx, Transactions, transactionRows); + await insertChunked(tx, Messages, messageRows); - for (const messageChunk of chunk(messageRows, INSERT_CHUNK_SIZE)) { - await tx.insert(Messages).values(messageChunk).onConflictDoNothing(); - } + await this.#balanceWriter.write(tx, balanceIntents); + await insertChunked(tx, AccountTxs, accountTxRows); await tx .insert(IndexerState) @@ -108,6 +121,52 @@ export class BlockCommitterService { }); } + /** + * Interns every address the batch touches — spenders, receivers, correlated counterparties and tx signers — + * on the base connection before the commit transaction, so the ledger and activity rows can reference their + * account ids by foreign key. + */ + async #internAccounts(balanceChanges: DerivedBalanceChange[], accountTxs: DerivedAccountTx[]): Promise> { + const addresses = new Set(); + + for (const change of balanceChanges) { + addresses.add(change.address); + if (change.counterpartyAddress) { + addresses.add(change.counterpartyAddress); + } + } + for (const row of accountTxs) { + addresses.add(row.address); + } + + return this.#interner.resolve(addresses); + } + + #resolveBalanceChanges(changes: DerivedBalanceChange[], accountIds: Map): ResolvedBalanceChange[] { + return changes.map(change => ({ + accountId: this.#requireId(accountIds, change.address), + counterpartyAccountId: change.counterpartyAddress ? accountIds.get(change.counterpartyAddress) ?? null : null, + denom: change.denom, + delta: change.delta, + reason: change.reason, + height: change.height, + txIndex: change.txIndex, + eventIndex: change.eventIndex + })); + } + + #resolveAccountTxs(rows: DerivedAccountTx[], accountIds: Map): (typeof AccountTxs.$inferInsert)[] { + return rows.map(row => ({ accountId: this.#requireId(accountIds, row.address), height: row.height, txIndex: row.txIndex, role: row.role })); + } + + #requireId(accountIds: Map, address: string): number { + const accountId = accountIds.get(address); + if (accountId === undefined) { + throw new Error(`No interned account id for address ${address}`); + } + return accountId; + } + async #internMessageTypes(blocks: DecodedBlock[]): Promise> { const typeUrls = new Set(blocks.flatMap(block => block.transactions.flatMap(tx => tx.messages.map(message => message.typeUrl)))); const uncached = [...typeUrls].filter(typeUrl => !this.#typeIds.has(typeUrl)); diff --git a/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts b/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts index ede66dbc3b..e5bee1a834 100644 --- a/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts +++ b/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts @@ -7,7 +7,7 @@ import { describe, expect, it } from "vitest"; import { envSchema } from "@src/config/env.config"; import { BlockDecoderService } from "@src/pipeline/block-decoder.service"; -import type { RpcBlockResult, RpcBlockResultsResult } from "@src/rpc/rpc-types"; +import type { RpcBlockResult, RpcBlockResultsResult, RpcEvent, RpcTxResult } from "@src/rpc/rpc-types"; describe(BlockDecoderService.name, () => { it("decodes block metadata with hashes as buffers", () => { @@ -81,6 +81,91 @@ describe(BlockDecoderService.name, () => { expect(decoded.transactions[0].code).toBe(11); }); + it("captures relevant tx events with their msg_index and drops irrelevant ones", () => { + const { decoder } = setup(); + const rawTx = buildMsgSendTx(); + const txResult: RpcTxResult = { + code: 0, + gas_used: "0", + gas_wanted: "0", + events: [ + event("coin_spent", { spender: "akash1from", amount: "42uakt", msg_index: "0" }), + event("tx", { fee: "5000uakt" }), + event("message", { action: "/cosmos.bank.v1beta1.MsgSend" }), + event("coin_received", { receiver: "akash1to", amount: "42uakt", msg_index: "0" }) + ] + }; + + const [tx] = decoder.decode(buildBlock({ txs: [rawTx.toString("base64")] }), buildBlockResults([txResult])).transactions; + + expect(tx.events).toEqual([ + { type: "coin_spent", attributes: { spender: "akash1from", amount: "42uakt", msg_index: "0" }, msgIndex: 0 }, + { type: "coin_received", attributes: { receiver: "akash1to", amount: "42uakt", msg_index: "0" }, msgIndex: 0 } + ]); + }); + + it("normalizes base64-encoded event attributes", () => { + const { decoder } = setup(); + const rawTx = buildMsgSendTx(); + const txResult: RpcTxResult = { + code: 0, + gas_used: "0", + gas_wanted: "0", + events: [event("coin_spent", { "c3BlbmRlcg==": "YWthc2gxZnJvbQ==" })] + }; + + const [tx] = decoder.decode(buildBlock({ txs: [rawTx.toString("base64")] }), buildBlockResults([txResult])).transactions; + + expect(tx.events[0].attributes).toEqual({ spender: "akash1from" }); + }); + + it("keeps a failed transaction's fee coin events", () => { + const { decoder } = setup(); + const rawTx = buildMsgSendTx(); + const txResult: RpcTxResult = { + code: 11, + gas_used: "70000", + gas_wanted: "70000", + events: [ + event("coin_spent", { spender: "akash1payer", amount: "5000uakt" }), + event("coin_received", { receiver: "akash1feecollector", amount: "5000uakt" }) + ] + }; + + const [tx] = decoder.decode(buildBlock({ txs: [rawTx.toString("base64")] }), buildBlockResults([txResult])).transactions; + + expect(tx.code).toBe(11); + expect(tx.events.map(e => e.type)).toEqual(["coin_spent", "coin_received"]); + }); + + it("prefers finalize_block_events for block-level events", () => { + const { decoder } = setup(); + + const decoded = decoder.decode( + buildBlock({ txs: [] }), + buildBlockResults([], { + finalize_block_events: [event("coinbase", { minter: "akash1mint", amount: "10uakt" })], + begin_block_events: [event("transfer", { sender: "ignored", recipient: "ignored", amount: "1uakt" })] + }) + ); + + expect(decoded.blockEvents).toEqual([{ type: "coinbase", attributes: { minter: "akash1mint", amount: "10uakt" } }]); + }); + + it("falls back to begin and end block events when finalize is absent", () => { + const { decoder } = setup(); + + const decoded = decoder.decode( + buildBlock({ txs: [] }), + buildBlockResults([], { + begin_block_events: [event("coin_received", { receiver: "akash1begin", amount: "1uakt" })], + end_block_events: [event("coin_spent", { spender: "akash1end", amount: "2uakt" })] + }) + ); + + expect(decoded.blockEvents.map(e => e.type)).toEqual(["coin_received", "coin_spent"]); + }); + function setup(input?: { maxBodyBytes?: number }) { const config = envSchema.parse({ POSTGRES_DB_URI: "postgres://unit:unit@localhost:5432/unit", @@ -105,8 +190,15 @@ describe(BlockDecoderService.name, () => { }; } - function buildBlockResults(txsResults: { code: number; gas_used: string; gas_wanted: string }[]): RpcBlockResultsResult { - return { height: "1234", txs_results: txsResults }; + function buildBlockResults( + txsResults: RpcTxResult[], + blockEvents?: Partial> + ): RpcBlockResultsResult { + return { height: "1234", txs_results: txsResults, ...blockEvents }; + } + + function event(type: string, attributes: Record): RpcEvent { + return { type, attributes: Object.entries(attributes).map(([key, value]) => ({ key, value })) }; } function buildMsgSendTx(typeUrl = "/cosmos.bank.v1beta1.MsgSend"): Buffer { diff --git a/apps/chain-indexer/src/pipeline/block-decoder.service.ts b/apps/chain-indexer/src/pipeline/block-decoder.service.ts index 987f68b935..1c9bed1ff8 100644 --- a/apps/chain-indexer/src/pipeline/block-decoder.service.ts +++ b/apps/chain-indexer/src/pipeline/block-decoder.service.ts @@ -5,11 +5,18 @@ import { inject, singleton } from "tsyringe"; import type { EnvConfig } from "@src/config/env.config"; import { toCanonicalJson } from "@src/pipeline/canonical-json"; -import type { DecodedBlock, DecodedMessage, DecodedTransaction } from "@src/pipeline/decoded-block"; +import { decodeIfBase64 } from "@src/pipeline/decode-if-base64"; +import type { DecodedBlock, DecodedEvent, DecodedMessage, DecodedTransaction } from "@src/pipeline/decoded-block"; +import { deriveSignerAddresses } from "@src/pipeline/signer-addresses"; import { APP_CONFIG } from "@src/providers/app-config.provider"; import type { Registry } from "@src/providers/type-registry.provider"; import { TYPE_REGISTRY } from "@src/providers/type-registry.provider"; -import type { RpcBlockResult, RpcBlockResultsResult, RpcTxResult } from "@src/rpc/rpc-types"; +import type { RpcBlockResult, RpcBlockResultsResult, RpcEvent, RpcTxResult } from "@src/rpc/rpc-types"; + +/** The ledger derives balances and reasons only from these event types; capturing the rest would waste memory across backfill batches. */ +const RELEVANT_EVENT_TYPES = new Set(["coin_spent", "coin_received", "transfer", "coinbase", "burn", "slash"]); + +const MSG_INDEX_ATTRIBUTE = "msg_index"; @singleton() export class BlockDecoderService { @@ -35,7 +42,8 @@ export class BlockDecoderService { hash: Buffer.from(block.block_id.hash, "hex"), parentHash: block.block.header.last_block_id?.hash ? Buffer.from(block.block.header.last_block_id.hash, "hex") : null, proposerAddress: block.block.header.proposer_address, - transactions: rawTxs.map((rawTx, index) => this.#decodeTransaction(rawTx, txResults[index], index)) + transactions: rawTxs.map((rawTx, index) => this.#decodeTransaction(rawTx, txResults[index], index)), + blockEvents: this.#decodeBlockEvents(blockResults) }; } @@ -50,10 +58,46 @@ export class BlockDecoderService { gasUsed: parseInt(txResult.gas_used ?? "0"), gasWanted: parseInt(txResult.gas_wanted ?? "0"), fee: decodedTx.authInfo.fee?.amount.map(({ denom, amount }) => ({ denom, amount })) ?? [], - messages: decodedTx.body.messages.map((message, messageIndex) => this.#decodeMessage(message, messageIndex)) + messages: decodedTx.body.messages.map((message, messageIndex) => this.#decodeMessage(message, messageIndex)), + events: this.#decodeEvents(txResult.events), + signerAddresses: deriveSignerAddresses(decodedTx.authInfo.signerInfos) }; } + /** + * ABCI 2.0 (CometBFT 0.38+) merges begin/end block events into `finalize_block_events`; older nodes split + * them. Mirrors the legacy indexer's `finalize_block_events ?? [...begin, ...end]` normalization. + */ + #decodeBlockEvents(blockResults: RpcBlockResultsResult): DecodedEvent[] { + const rawEvents = blockResults.finalize_block_events ?? [...(blockResults.begin_block_events ?? []), ...(blockResults.end_block_events ?? [])]; + return this.#decodeEvents(rawEvents); + } + + #decodeEvents(rawEvents: RpcEvent[] | undefined): DecodedEvent[] { + if (!rawEvents) { + return []; + } + + return rawEvents.filter(event => RELEVANT_EVENT_TYPES.has(event.type)).map(event => this.#decodeEvent(event)); + } + + #decodeEvent(event: RpcEvent): DecodedEvent { + const attributes: Record = {}; + let msgIndex: number | undefined; + + for (const attribute of event.attributes) { + const key = decodeIfBase64(attribute.key); + const value = attribute.value ? decodeIfBase64(attribute.value) : ""; + attributes[key] = value; + + if (key === MSG_INDEX_ATTRIBUTE) { + msgIndex = parseInt(value); + } + } + + return msgIndex === undefined ? { type: event.type, attributes } : { type: event.type, attributes, msgIndex }; + } + #decodeMessage(message: { typeUrl: string; value: Uint8Array }, index: number): DecodedMessage { return { index, diff --git a/apps/chain-indexer/src/pipeline/decode-if-base64.spec.ts b/apps/chain-indexer/src/pipeline/decode-if-base64.spec.ts new file mode 100644 index 0000000000..f87d075eda --- /dev/null +++ b/apps/chain-indexer/src/pipeline/decode-if-base64.spec.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; + +import { decodeIfBase64 } from "@src/pipeline/decode-if-base64"; + +describe("decodeIfBase64", () => { + it("decodes a base64-encoded printable-ascii value", () => { + expect(decodeIfBase64("c3BlbmRlcg==")).toBe("spender"); + }); + + it("returns an already-plaintext value untouched", () => { + expect(decodeIfBase64("spender")).toBe("spender"); + }); + + it("returns a plaintext value that happens to be valid base64 untouched when it stays printable", () => { + expect(decodeIfBase64("akash1abcd")).toBe("akash1abcd"); + }); + + it("leaves an empty string untouched", () => { + expect(decodeIfBase64("")).toBe(""); + }); + + it("returns a value whose length is not a multiple of four untouched", () => { + expect(decodeIfBase64("abc")).toBe("abc"); + }); + + it("keeps a value that decodes to non-printable bytes as the original", () => { + expect(decodeIfBase64("////")).toBe("////"); + }); +}); diff --git a/apps/chain-indexer/src/pipeline/decode-if-base64.ts b/apps/chain-indexer/src/pipeline/decode-if-base64.ts new file mode 100644 index 0000000000..26dd4ea6a2 --- /dev/null +++ b/apps/chain-indexer/src/pipeline/decode-if-base64.ts @@ -0,0 +1,35 @@ +const BASE64_PATTERN = /^[A-Za-z0-9+/]+={0,2}$/; + +function isPrintableAscii(value: string): boolean { + for (let i = 0; i < value.length; i++) { + const code = value.charCodeAt(i); + if (code < 32 || code > 126) { + return false; + } + } + return true; +} + +/** + * ABCI event attributes arrive base64-encoded on some CometBFT versions and plaintext on others, so the + * decoder normalizes every key/value through this guard. A value is only decoded when it round-trips + * through base64 exactly and yields printable ASCII, which keeps genuine plaintext (even plaintext that + * happens to be valid base64) untouched. Ported from the legacy indexer to stay node-version agnostic. + */ +export function decodeIfBase64(value: string): string { + if (!value || value.length % 4 !== 0 || !BASE64_PATTERN.test(value)) { + return value; + } + + try { + const decoded = atob(value); + + if (btoa(decoded) !== value) { + return value; + } + + return isPrintableAscii(decoded) ? decoded : value; + } catch { + return value; + } +} diff --git a/apps/chain-indexer/src/pipeline/decoded-block.ts b/apps/chain-indexer/src/pipeline/decoded-block.ts index c1225790a0..51acfb247a 100644 --- a/apps/chain-indexer/src/pipeline/decoded-block.ts +++ b/apps/chain-indexer/src/pipeline/decoded-block.ts @@ -6,6 +6,13 @@ export interface DecodedMessage { body: unknown | null; } +/** An ABCI event with base64-normalized attributes flattened to a key→value map. `msgIndex` links the event to its message where present. */ +export interface DecodedEvent { + type: string; + attributes: Record; + msgIndex?: number; +} + export interface DecodedTransaction { index: number; hash: Buffer; @@ -14,6 +21,8 @@ export interface DecodedTransaction { gasWanted: number; fee: FeeCoin[]; messages: DecodedMessage[]; + events: DecodedEvent[]; + signerAddresses: string[]; } export interface DecodedBlock { @@ -23,4 +32,5 @@ export interface DecodedBlock { parentHash: Buffer | null; proposerAddress: string; transactions: DecodedTransaction[]; + blockEvents: DecodedEvent[]; } diff --git a/apps/chain-indexer/src/pipeline/signer-addresses.spec.ts b/apps/chain-indexer/src/pipeline/signer-addresses.spec.ts new file mode 100644 index 0000000000..45bb11eea8 --- /dev/null +++ b/apps/chain-indexer/src/pipeline/signer-addresses.spec.ts @@ -0,0 +1,38 @@ +import { createMultisigThresholdPubkey, encodeSecp256k1Pubkey, pubkeyToAddress } from "@cosmjs/amino"; +import { encodePubkey } from "@cosmjs/proto-signing"; +import { SignerInfo } from "cosmjs-types/cosmos/tx/v1beta1/tx"; +import { describe, expect, it } from "vitest"; + +import { AKASH_ADDRESS_PREFIX } from "@src/genesis/genesis-address"; +import { deriveSignerAddresses } from "@src/pipeline/signer-addresses"; + +function secp256k1Pubkey(firstByte: number) { + return encodeSecp256k1Pubkey(new Uint8Array([firstByte, ...new Array(32).fill(1)])); +} + +function signerInfoWith(pubkey: Parameters[0]) { + return SignerInfo.fromPartial({ publicKey: encodePubkey(pubkey) }); +} + +describe("deriveSignerAddresses", () => { + it("derives the account address of a single secp256k1 signer", () => { + const pubkey = secp256k1Pubkey(2); + + expect(deriveSignerAddresses([signerInfoWith(pubkey)])).toEqual([pubkeyToAddress(pubkey, AKASH_ADDRESS_PREFIX)]); + }); + + it("derives every member address of a multisig signer", () => { + const first = secp256k1Pubkey(2); + const second = secp256k1Pubkey(3); + const multisig = createMultisigThresholdPubkey([first, second], 1); + + expect(deriveSignerAddresses([signerInfoWith(multisig)])).toEqual([ + pubkeyToAddress(first, AKASH_ADDRESS_PREFIX), + pubkeyToAddress(second, AKASH_ADDRESS_PREFIX) + ]); + }); + + it("skips signer infos without a public key", () => { + expect(deriveSignerAddresses([SignerInfo.fromPartial({})])).toEqual([]); + }); +}); diff --git a/apps/chain-indexer/src/pipeline/signer-addresses.ts b/apps/chain-indexer/src/pipeline/signer-addresses.ts new file mode 100644 index 0000000000..b0de801bb6 --- /dev/null +++ b/apps/chain-indexer/src/pipeline/signer-addresses.ts @@ -0,0 +1,44 @@ +import type { Pubkey, SinglePubkey } from "@cosmjs/amino"; +import { isMultisigThresholdPubkey, isSinglePubkey, pubkeyToAddress } from "@cosmjs/amino"; +import { decodePubkey } from "@cosmjs/proto-signing"; +import type { SignerInfo } from "cosmjs-types/cosmos/tx/v1beta1/tx"; + +import { AKASH_ADDRESS_PREFIX } from "@src/genesis/genesis-address"; + +function flattenSinglePubkeys(pubkey: Pubkey): SinglePubkey[] { + if (isMultisigThresholdPubkey(pubkey)) { + return pubkey.value.pubkeys.flatMap(flattenSinglePubkeys); + } + + return isSinglePubkey(pubkey) ? [pubkey] : []; +} + +/** + * Bech32 account addresses of every signer of a transaction. A multisig signer expands to one address per + * member key, mirroring the legacy indexer. An undecodable pubkey (e.g. a legacy amino multisig) yields no + * address for that signer rather than failing the whole block. + */ +export function deriveSignerAddresses(signerInfos: readonly SignerInfo[]): string[] { + const addresses: string[] = []; + + for (const signerInfo of signerInfos) { + if (!signerInfo.publicKey) { + continue; + } + + try { + const pubkey = decodePubkey(signerInfo.publicKey); + if (!pubkey) { + continue; + } + + for (const single of flattenSinglePubkeys(pubkey)) { + addresses.push(pubkeyToAddress(single, AKASH_ADDRESS_PREFIX)); + } + } catch { + continue; + } + } + + return addresses; +} diff --git a/apps/chain-indexer/src/reconcile/bank-query.spec.ts b/apps/chain-indexer/src/reconcile/bank-query.spec.ts new file mode 100644 index 0000000000..0944d3ad25 --- /dev/null +++ b/apps/chain-indexer/src/reconcile/bank-query.spec.ts @@ -0,0 +1,44 @@ +import { toBase64 } from "@cosmjs/encoding"; +import { QueryAllBalancesResponse, QueryTotalSupplyResponse } from "cosmjs-types/cosmos/bank/v1beta1/query"; +import { describe, expect, it } from "vitest"; + +import { + ALL_BALANCES_PATH, + decodeAllBalances, + decodeTotalSupply, + encodeAllBalancesRequest, + encodeTotalSupplyRequest, + TOTAL_SUPPLY_PATH +} from "@src/reconcile/bank-query"; + +describe("bank-query", () => { + it("encodes an all-balances request as hex carrying the address", () => { + const hex = encodeAllBalancesRequest("akash1abc"); + + expect(hex).toMatch(/^[0-9a-f]+$/); + expect(Buffer.from(hex, "hex").toString("utf8")).toContain("akash1abc"); + expect(ALL_BALANCES_PATH).toBe("/cosmos.bank.v1beta1.Query/AllBalances"); + }); + + it("encodes a total-supply request as hex", () => { + expect(encodeTotalSupplyRequest()).toMatch(/^[0-9a-f]*$/); + expect(TOTAL_SUPPLY_PATH).toBe("/cosmos.bank.v1beta1.Query/TotalSupply"); + }); + + it("decodes an all-balances response into typed coins", () => { + const value = toBase64(QueryAllBalancesResponse.encode({ balances: [{ denom: "uakt", amount: "42" }], pagination: undefined }).finish()); + + expect(decodeAllBalances(value)).toEqual([{ denom: "uakt", amount: 42n }]); + }); + + it("decodes a total-supply response into typed coins", () => { + const value = toBase64(QueryTotalSupplyResponse.encode({ supply: [{ denom: "uakt", amount: "1000" }], pagination: undefined }).finish()); + + expect(decodeTotalSupply(value)).toEqual([{ denom: "uakt", amount: 1000n }]); + }); + + it("decodes an empty value as no coins", () => { + expect(decodeAllBalances(null)).toEqual([]); + expect(decodeTotalSupply(null)).toEqual([]); + }); +}); diff --git a/apps/chain-indexer/src/reconcile/bank-query.ts b/apps/chain-indexer/src/reconcile/bank-query.ts new file mode 100644 index 0000000000..0a3770f6ad --- /dev/null +++ b/apps/chain-indexer/src/reconcile/bank-query.ts @@ -0,0 +1,40 @@ +import { fromBase64, toHex } from "@cosmjs/encoding"; +import { QueryAllBalancesRequest, QueryAllBalancesResponse, QueryTotalSupplyRequest, QueryTotalSupplyResponse } from "cosmjs-types/cosmos/bank/v1beta1/query"; + +import type { CoinAmount } from "@src/pipeline/balance/coin-amount"; + +export const ALL_BALANCES_PATH = "/cosmos.bank.v1beta1.Query/AllBalances"; +export const TOTAL_SUPPLY_PATH = "/cosmos.bank.v1beta1.Query/TotalSupply"; + +/** Cosmos paginates bank queries; a single large page covers any account's handful of denoms and the chain's denom set. */ +const PAGE_LIMIT = 10_000n; + +export function encodeAllBalancesRequest(address: string): string { + return toHex(QueryAllBalancesRequest.encode(QueryAllBalancesRequest.fromPartial({ address, pagination: pageRequest() })).finish()); +} + +export function encodeTotalSupplyRequest(): string { + return toHex(QueryTotalSupplyRequest.encode(QueryTotalSupplyRequest.fromPartial({ pagination: pageRequest() })).finish()); +} + +export function decodeAllBalances(value: string | null): CoinAmount[] { + if (!value) { + return []; + } + return toCoinAmounts(QueryAllBalancesResponse.decode(fromBase64(value)).balances); +} + +export function decodeTotalSupply(value: string | null): CoinAmount[] { + if (!value) { + return []; + } + return toCoinAmounts(QueryTotalSupplyResponse.decode(fromBase64(value)).supply); +} + +function pageRequest() { + return { key: new Uint8Array(), offset: 0n, limit: PAGE_LIMIT, countTotal: false, reverse: false }; +} + +function toCoinAmounts(coins: { denom: string; amount: string }[]): CoinAmount[] { + return coins.map(coin => ({ denom: coin.denom, amount: BigInt(coin.amount) })); +} diff --git a/apps/chain-indexer/src/reconcile/coin-diff.spec.ts b/apps/chain-indexer/src/reconcile/coin-diff.spec.ts new file mode 100644 index 0000000000..951f7dfe27 --- /dev/null +++ b/apps/chain-indexer/src/reconcile/coin-diff.spec.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from "vitest"; + +import { diffCoins } from "@src/reconcile/coin-diff"; + +describe("diffCoins", () => { + it("returns no differences when both sides match", () => { + expect(diffCoins([{ denom: "uakt", amount: 100n }], [{ denom: "uakt", amount: 100n }])).toEqual([]); + }); + + it("reports a denom whose amounts differ", () => { + expect(diffCoins([{ denom: "uakt", amount: 100n }], [{ denom: "uakt", amount: 90n }])).toEqual([{ denom: "uakt", expected: 100n, actual: 90n }]); + }); + + it("reports a denom present only on the chain as a zero ledger balance", () => { + expect(diffCoins([{ denom: "uakt", amount: 5n }], [])).toEqual([{ denom: "uakt", expected: 5n, actual: 0n }]); + }); + + it("reports a denom present only in the ledger as a zero chain balance", () => { + expect(diffCoins([], [{ denom: "uakt", amount: 5n }])).toEqual([{ denom: "uakt", expected: 0n, actual: 5n }]); + }); + + it("orders differences by denom", () => { + expect( + diffCoins( + [ + { denom: "uosmo", amount: 1n }, + { denom: "uakt", amount: 1n } + ], + [] + ).map(diff => diff.denom) + ).toEqual(["uakt", "uosmo"]); + }); +}); diff --git a/apps/chain-indexer/src/reconcile/coin-diff.ts b/apps/chain-indexer/src/reconcile/coin-diff.ts new file mode 100644 index 0000000000..eb8697dbb1 --- /dev/null +++ b/apps/chain-indexer/src/reconcile/coin-diff.ts @@ -0,0 +1,30 @@ +import type { CoinAmount } from "@src/pipeline/balance/coin-amount"; + +/** A denom whose chain-queried (`expected`) and ledger-derived (`actual`) amounts disagree. */ +export interface CoinDiff { + denom: string; + expected: bigint; + actual: bigint; +} + +function toMap(coins: CoinAmount[]): Map { + return new Map(coins.map(coin => [coin.denom, coin.amount])); +} + +/** Compares chain-queried balances (`expected`) against ledger-derived balances (`actual`), returning only the denoms that differ. */ +export function diffCoins(expected: CoinAmount[], actual: CoinAmount[]): CoinDiff[] { + const expectedByDenom = toMap(expected); + const actualByDenom = toMap(actual); + const denoms = [...new Set([...expectedByDenom.keys(), ...actualByDenom.keys()])].sort(); + + const diffs: CoinDiff[] = []; + for (const denom of denoms) { + const expectedAmount = expectedByDenom.get(denom) ?? 0n; + const actualAmount = actualByDenom.get(denom) ?? 0n; + if (expectedAmount !== actualAmount) { + diffs.push({ denom, expected: expectedAmount, actual: actualAmount }); + } + } + + return diffs; +} diff --git a/apps/chain-indexer/src/reconcile/reconcile.service.spec.ts b/apps/chain-indexer/src/reconcile/reconcile.service.spec.ts new file mode 100644 index 0000000000..5b28f62d37 --- /dev/null +++ b/apps/chain-indexer/src/reconcile/reconcile.service.spec.ts @@ -0,0 +1,132 @@ +import { toBase64 } from "@cosmjs/encoding"; +import { QueryAllBalancesResponse, QueryTotalSupplyResponse } from "cosmjs-types/cosmos/bank/v1beta1/query"; +import { describe, expect, it, vi } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import { IndexerState } from "@src/db/schema"; +import type { ChainDatabase } from "@src/providers/db.provider"; +import type { LoggerService } from "@src/providers/logging.provider"; +import { ALL_BALANCES_PATH } from "@src/reconcile/bank-query"; +import { ReconcileService } from "@src/reconcile/reconcile.service"; +import type { RpcClientPool } from "@src/rpc/rpc-client-pool.service"; + +const coin = (denom: string, amount: string) => ({ denom, amount }); + +describe(ReconcileService.name, () => { + it("returns true when every sampled account and the total supply match the chain", async () => { + const { service, abciQuery } = setup({ + checkpoint: 100, + balanceRows: [{ address: "akash1a", denom: "uakt", amount: "100" }], + chainBalances: { akash1a: [coin("uakt", "100")] }, + chainSupply: [coin("uakt", "100")] + }); + + await expect(service.reconcile()).resolves.toBe(true); + expect(abciQuery.mock.calls.every(call => call[2] === 100)).toBe(true); + }); + + it("returns false when a sampled account balance disagrees with the chain", async () => { + const { service } = setup({ + checkpoint: 100, + balanceRows: [{ address: "akash1a", denom: "uakt", amount: "100" }], + chainBalances: { akash1a: [coin("uakt", "90")] }, + chainSupply: [coin("uakt", "100")] + }); + + await expect(service.reconcile()).resolves.toBe(false); + }); + + it("returns false when the total supply disagrees with the ledger", async () => { + const { service } = setup({ + checkpoint: 100, + balanceRows: [{ address: "akash1a", denom: "uakt", amount: "100" }], + chainBalances: { akash1a: [coin("uakt", "100")] }, + chainSupply: [coin("uakt", "999")] + }); + + await expect(service.reconcile()).resolves.toBe(false); + }); + + it("returns false when there is no sync checkpoint to reconcile against", async () => { + const { service } = setup({ checkpoint: undefined, balanceRows: [], chainBalances: {}, chainSupply: [] }); + + await expect(service.reconcile()).resolves.toBe(false); + }); + + it("fails fast on a non-integer sample size instead of silently checking nothing", async () => { + const { service, abciQuery } = setup({ + checkpoint: 100, + balanceRows: [{ address: "akash1a", denom: "uakt", amount: "100" }], + chainBalances: { akash1a: [coin("uakt", "100")] }, + chainSupply: [coin("uakt", "100")] + }); + + await expect(service.reconcile({ sampleSize: NaN })).resolves.toBe(false); + expect(abciQuery).not.toHaveBeenCalled(); + }); + + it("reads the checkpoint height and ledger balances from one repeatable-read snapshot", async () => { + const { service, transaction, baseSelect, txSelect } = setup({ + checkpoint: 100, + balanceRows: [{ address: "akash1a", denom: "uakt", amount: "100" }], + chainBalances: { akash1a: [coin("uakt", "100")] }, + chainSupply: [coin("uakt", "100")] + }); + + await service.reconcile(); + + expect(transaction).toHaveBeenCalledWith(expect.any(Function), { isolationLevel: "repeatable read", accessMode: "read only" }); + expect(baseSelect).not.toHaveBeenCalled(); + expect(txSelect).toHaveBeenCalledTimes(2); + }); + + it("caps concurrent balance queries instead of issuing them one at a time", async () => { + const balanceRows = Array.from({ length: 25 }, (_, index) => ({ address: `akash1a${index}`, denom: "uakt", amount: "100" })); + const chainBalances = Object.fromEntries(balanceRows.map(row => [row.address, [coin("uakt", "100")]])); + const { service, maxInFlight } = setup({ checkpoint: 100, balanceRows, chainBalances, chainSupply: [coin("uakt", "2500")] }); + + await service.reconcile(); + + expect(maxInFlight()).toBeGreaterThan(1); + expect(maxInFlight()).toBeLessThan(balanceRows.length); + }); + + function setup(input: { + checkpoint: number | undefined; + balanceRows: { address: string; denom: string; amount: string }[]; + chainBalances: Record; + chainSupply: { denom: string; amount: string }[]; + }) { + const buildSelect = () => ({ + from: (table: unknown) => { + if (table === IndexerState) { + return { where: () => Promise.resolve(input.checkpoint === undefined ? [] : [{ lastHeight: input.checkpoint }]) }; + } + return { innerJoin: () => Promise.resolve(input.balanceRows) }; + } + }); + + const baseSelect = vi.fn(buildSelect); + const txSelect = vi.fn(buildSelect); + const transaction = vi.fn(async (callback: (tx: unknown) => unknown) => callback({ select: txSelect })); + const dbFake = { select: baseSelect, transaction }; + + let inFlight = 0; + let maxInFlight = 0; + const rpc = mock(); + rpc.abciQuery.mockImplementation(async (path, dataHex) => { + if (path === ALL_BALANCES_PATH) { + inFlight++; + maxInFlight = Math.max(maxInFlight, inFlight); + await Promise.resolve(); + inFlight--; + const address = Object.keys(input.chainBalances).find(candidate => Buffer.from(dataHex, "hex").toString("utf8").includes(candidate)); + return { value: toBase64(QueryAllBalancesResponse.encode({ balances: input.chainBalances[address ?? ""] ?? [], pagination: undefined }).finish()) }; + } + return { value: toBase64(QueryTotalSupplyResponse.encode({ supply: input.chainSupply, pagination: undefined }).finish()) }; + }); + + const service = new ReconcileService(dbFake as unknown as ChainDatabase, rpc, mock()); + return { service, abciQuery: rpc.abciQuery, transaction, baseSelect, txSelect, maxInFlight: () => maxInFlight }; + } +}); diff --git a/apps/chain-indexer/src/reconcile/reconcile.service.ts b/apps/chain-indexer/src/reconcile/reconcile.service.ts new file mode 100644 index 0000000000..ff93a332e9 --- /dev/null +++ b/apps/chain-indexer/src/reconcile/reconcile.service.ts @@ -0,0 +1,167 @@ +import { eq } from "drizzle-orm"; +import { inject, singleton } from "tsyringe"; + +import { AccountBalances, Accounts, IndexerState } from "@src/db/schema"; +import type { CoinAmount } from "@src/pipeline/balance/coin-amount"; +import { SYNC_STREAM } from "@src/pipeline/block-committer.service"; +import type { ChainDatabase, ChainTransaction } from "@src/providers/db.provider"; +import { CHAIN_DB } from "@src/providers/db.provider"; +import { LoggerService } from "@src/providers/logging.provider"; +import { + ALL_BALANCES_PATH, + decodeAllBalances, + decodeTotalSupply, + encodeAllBalancesRequest, + encodeTotalSupplyRequest, + TOTAL_SUPPLY_PATH +} from "@src/reconcile/bank-query"; +import type { CoinDiff } from "@src/reconcile/coin-diff"; +import { diffCoins } from "@src/reconcile/coin-diff"; +import { RpcClientPool } from "@src/rpc/rpc-client-pool.service"; + +const DEFAULT_SAMPLE_SIZE = 100; + +/** Bounds concurrent ABCI round-trips so the sampled accounts load-balance across the RPC pool without flooding any single node. */ +const RECONCILE_CONCURRENCY = 10; + +interface AccountBalance { + address: string; + coins: CoinAmount[]; +} + +/** + * Proves the ledger matches the chain. At the indexer's `sync` checkpoint height it compares each sampled + * account's current balance against the node's bank balance, and the ledger's total per denom against the + * chain's supply. Querying at the checkpoint (not the moving tip) keeps the comparison race-free. + */ +@singleton() +export class ReconcileService { + readonly #db: ChainDatabase; + readonly #rpc: RpcClientPool; + readonly #logger: LoggerService; + + constructor(@inject(CHAIN_DB) db: ChainDatabase, @inject(RpcClientPool) rpc: RpcClientPool, @inject(LoggerService) logger: LoggerService) { + this.#db = db; + this.#rpc = rpc; + this.#logger = logger; + this.#logger.setContext("RECONCILE"); + } + + async reconcile({ sampleSize = DEFAULT_SAMPLE_SIZE }: { sampleSize?: number } = {}): Promise { + if (!Number.isInteger(sampleSize) || sampleSize <= 0) { + this.#logger.error({ event: "RECONCILE_INVALID_SAMPLE_SIZE", sampleSize }); + return false; + } + + const snapshot = await this.#readSnapshot(); + if (snapshot === undefined) { + this.#logger.warn({ event: "RECONCILE_NO_CHECKPOINT" }); + return false; + } + + const { height, balances } = snapshot; + const sampled = this.#sample(balances, sampleSize); + this.#logger.info({ event: "RECONCILE_START", height, accounts: balances.length, sampled: sampled.length }); + + const accountMatches = await mapWithConcurrency(sampled, RECONCILE_CONCURRENCY, async account => { + const chain = decodeAllBalances((await this.#rpc.abciQuery(ALL_BALANCES_PATH, encodeAllBalancesRequest(account.address), height)).value); + const diffs = diffCoins(chain, account.coins); + if (diffs.length === 0) return true; + this.#logger.error({ event: "RECONCILE_ACCOUNT_MISMATCH", address: account.address, diffs: format(diffs) }); + return false; + }); + + let mismatches = accountMatches.filter(matched => !matched).length; + + const chainSupply = decodeTotalSupply((await this.#rpc.abciQuery(TOTAL_SUPPLY_PATH, encodeTotalSupplyRequest(), height)).value); + const supplyDiffs = diffCoins(chainSupply, totals(balances)); + if (supplyDiffs.length > 0) { + mismatches++; + this.#logger.error({ event: "RECONCILE_SUPPLY_MISMATCH", diffs: format(supplyDiffs) }); + } + + const ok = mismatches === 0; + this.#logger[ok ? "info" : "error"]({ event: ok ? "RECONCILE_OK" : "RECONCILE_FAILED", height, mismatches }); + return ok; + } + + /** + * Reads the checkpoint height and the ledger balances from one REPEATABLE READ snapshot. The committer advances + * both in a single transaction, so reading them as two independent SELECTs could straddle a commit — stale height + * against post-commit balances — and flag spurious mismatches on any account touched by that block. + */ + async #readSnapshot(): Promise<{ height: number; balances: AccountBalance[] } | undefined> { + return this.#db.transaction( + async tx => { + const height = await this.#readCheckpointHeight(tx); + if (height === undefined) return undefined; + return { height, balances: await this.#readLedgerBalances(tx) }; + }, + { isolationLevel: "repeatable read", accessMode: "read only" } + ); + } + + async #readCheckpointHeight(tx: ChainTransaction): Promise { + const [row] = await tx.select().from(IndexerState).where(eq(IndexerState.stream, SYNC_STREAM)); + return row?.lastHeight; + } + + async #readLedgerBalances(tx: ChainTransaction): Promise { + const rows = await tx + .select({ address: Accounts.address, denom: AccountBalances.denom, amount: AccountBalances.amount }) + .from(AccountBalances) + .innerJoin(Accounts, eq(AccountBalances.accountId, Accounts.id)); + + const byAddress = new Map(); + for (const row of rows) { + const coins = byAddress.get(row.address) ?? []; + coins.push({ denom: row.denom, amount: BigInt(row.amount) }); + byAddress.set(row.address, coins); + } + + return [...byAddress.entries()].map(([address, coins]) => ({ address, coins })); + } + + /** Samples the highest-balance accounts, which carry the most reconciliation signal, capping RPC round-trips at `sampleSize`. */ + #sample(balances: AccountBalance[], sampleSize: number): AccountBalance[] { + return balances + .map(account => ({ account, total: sumCoins(account.coins) })) + .sort((a, b) => (b.total < a.total ? -1 : 1)) + .slice(0, sampleSize) + .map(entry => entry.account); + } +} + +/** Runs `worker` over `items` with at most `limit` in flight at once, preserving input order in the returned results. */ +async function mapWithConcurrency(items: T[], limit: number, worker: (item: T) => Promise): Promise { + const results = new Array(items.length); + let next = 0; + + async function runWorker(): Promise { + while (next < items.length) { + const index = next++; + results[index] = await worker(items[index]); + } + } + + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, runWorker)); + return results; +} + +function totals(balances: AccountBalance[]): CoinAmount[] { + const byDenom = new Map(); + for (const account of balances) { + for (const coin of account.coins) { + byDenom.set(coin.denom, (byDenom.get(coin.denom) ?? 0n) + coin.amount); + } + } + return [...byDenom.entries()].map(([denom, amount]) => ({ denom, amount })); +} + +function sumCoins(coins: CoinAmount[]): bigint { + return coins.reduce((sum, coin) => sum + coin.amount, 0n); +} + +function format(diffs: CoinDiff[]): { denom: string; expected: string; actual: string }[] { + return diffs.map(diff => ({ denom: diff.denom, expected: diff.expected.toString(), actual: diff.actual.toString() })); +} diff --git a/apps/chain-indexer/src/reconcile/reconcile.ts b/apps/chain-indexer/src/reconcile/reconcile.ts new file mode 100644 index 0000000000..3a2d9a98f3 --- /dev/null +++ b/apps/chain-indexer/src/reconcile/reconcile.ts @@ -0,0 +1,37 @@ +import "@src/providers"; + +import { createOtelLogger } from "@akashnetwork/logging/otel"; +import { container } from "tsyringe"; + +import { envSchema } from "@src/config/env.config"; +import { PgClientService } from "@src/db/pg-client.service"; +import { ReconcileService } from "@src/reconcile/reconcile.service"; + +/** + * One-shot reconciliation entrypoint (`npm run reconcile`): exits 0 when the ledger matches the chain at the + * sync checkpoint height, non-zero on any mismatch or misconfiguration, so it can gate a deploy. + */ +async function main(): Promise { + const logger = createOtelLogger({ context: "RECONCILE_CLI" }); + + const parsed = envSchema.safeParse(process.env); + if (!parsed.success) { + logger.error({ event: "CONFIG_INVALID", issues: parsed.error.issues.map(issue => ({ path: issue.path.join(".") || "(root)", message: issue.message })) }); + process.exitCode = 1; + return; + } + + const sampleSize = parsed.data.RECONCILE_SAMPLE_SIZE; + + try { + const ok = await container.resolve(ReconcileService).reconcile(sampleSize === undefined ? {} : { sampleSize }); + process.exitCode = ok ? 0 : 1; + } catch (error) { + logger.error({ event: "RECONCILE_FATAL", error }); + process.exitCode = 1; + } finally { + await container.resolve(PgClientService).dispose(); + } +} + +void main(); diff --git a/apps/chain-indexer/src/rpc/rpc-client-pool.service.spec.ts b/apps/chain-indexer/src/rpc/rpc-client-pool.service.spec.ts index 6561c5120d..ec68268ff2 100644 --- a/apps/chain-indexer/src/rpc/rpc-client-pool.service.spec.ts +++ b/apps/chain-indexer/src/rpc/rpc-client-pool.service.spec.ts @@ -98,6 +98,42 @@ describe(RpcClientPool.name, () => { expect(fetchMock.mock.calls[0][0]).toBe("http://node-a/genesis_chunked?chunk=2"); }); + it("runs an abci query with a quoted path, hex data and historical height", async () => { + const { pool, fetchMock } = setup(); + fetchMock.mockResolvedValue(jsonResponse({ result: { response: { code: 0, value: "AA==" } } })); + + const response = await pool.abciQuery("/cosmos.bank.v1beta1.Query/TotalSupply", "0a00", 100); + + expect(response.value).toBe("AA=="); + expect(fetchMock.mock.calls[0][0]).toBe( + "http://node-a/abci_query?path=%22%2Fcosmos.bank.v1beta1.Query%2FTotalSupply%22&data=0x0a00&height=100&prove=false" + ); + }); + + it("fails over to the next node when a node answers with a non-zero abci code", async () => { + const { pool, fetchMock } = setup(); + fetchMock.mockResolvedValueOnce(jsonResponse({ result: { response: { code: 26, log: "height not available", value: null } } })); + fetchMock.mockResolvedValueOnce(jsonResponse({ result: { response: { code: 0, value: "AA==" } } })); + + const response = await pool.abciQuery("/cosmos.bank.v1beta1.Query/AllBalances", "00", 999); + + expect(response.value).toBe("AA=="); + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(fetchMock.mock.calls[0][0]).toContain("http://node-a/"); + expect(fetchMock.mock.calls[1][0]).toContain("http://node-b/"); + }); + + it("throws an aggregate error carrying the abci log when every node answers with a non-zero code", async () => { + const { pool, fetchMock } = setup(); + fetchMock.mockResolvedValue(jsonResponse({ result: { response: { code: 26, log: "height not available", value: null } } })); + + const error = await pool.abciQuery("/cosmos.bank.v1beta1.Query/AllBalances", "00", 999).catch(caught => caught); + + expect(error).toBeInstanceOf(AggregateError); + expect(error.errors[0].message).toContain("height not available"); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + function setup() { const fetchMock = vi.fn(); vi.stubGlobal("fetch", fetchMock); diff --git a/apps/chain-indexer/src/rpc/rpc-client-pool.service.ts b/apps/chain-indexer/src/rpc/rpc-client-pool.service.ts index 3cee43a144..ee387a164d 100644 --- a/apps/chain-indexer/src/rpc/rpc-client-pool.service.ts +++ b/apps/chain-indexer/src/rpc/rpc-client-pool.service.ts @@ -4,7 +4,7 @@ import { inject, singleton } from "tsyringe"; import type { EnvConfig } from "@src/config/env.config"; import { APP_CONFIG } from "@src/providers/app-config.provider"; import { LoggerService } from "@src/providers/logging.provider"; -import type { RpcBlockResult, RpcBlockResultsResult, RpcGenesisChunkResult, RpcStatusResult } from "@src/rpc/rpc-types"; +import type { RpcAbciQueryResult, RpcBlockResult, RpcBlockResultsResult, RpcGenesisChunkResult, RpcStatusResult } from "@src/rpc/rpc-types"; interface RpcNodeState { endpoint: string; @@ -63,13 +63,35 @@ export class RpcClientPool { return await this.#get(`/genesis_chunked?chunk=${chunk}`); } - async #get(path: string): Promise { + /** + * Runs an ABCI query against historical state at `height`. Reconciliation reads bank balances at the + * indexer's checkpoint height (not the moving tip), which requires an unpruned node — sandbox is archival. + */ + async abciQuery(path: string, dataHex: string, height: number): Promise { + const result = await this.#get( + `/abci_query?path=${encodeURIComponent(`"${path}"`)}&data=0x${dataHex}&height=${height}&prove=false`, + result => { + if (result.response.code) { + throw new Error(`abci_query ${path} failed at height ${height}: ${result.response.log ?? `code ${result.response.code}`}`); + } + } + ); + + return result.response; + } + + /** + * A `validate` failure is treated like a transport failure so the failover loop tries the next node: a pruned + * node answering HTTP 200 with a non-zero ABCI code (e.g. "height not available") must fail over to an archival one. + */ + async #get(path: string, validate?: (result: T) => void): Promise { const errors: unknown[] = []; for (const node of this.#candidates()) { node.inFlight++; try { const result = await this.#fetchFromNode(node.endpoint, path); + validate?.(result); node.unhealthyUntil = 0; return result; } catch (error) { diff --git a/apps/chain-indexer/src/rpc/rpc-types.ts b/apps/chain-indexer/src/rpc/rpc-types.ts index 60fbec7b23..f0a8bc051d 100644 --- a/apps/chain-indexer/src/rpc/rpc-types.ts +++ b/apps/chain-indexer/src/rpc/rpc-types.ts @@ -26,17 +26,37 @@ export interface RpcBlockResult { }; } +/** An ABCI event. Attribute keys/values may be base64-encoded depending on the CometBFT version, so callers normalize them. */ +export interface RpcEvent { + type: string; + attributes: { key: string; value: string | null }[]; +} + /** Fields marshaled with proto3 omitempty semantics may be absent when zero (e.g. code 0 on success). */ export interface RpcTxResult { code?: number; log?: string; gas_used?: string; gas_wanted?: string; + events?: RpcEvent[]; } export interface RpcBlockResultsResult { height: string; txs_results: RpcTxResult[] | null; + finalize_block_events?: RpcEvent[]; + begin_block_events?: RpcEvent[]; + end_block_events?: RpcEvent[]; +} + +/** CometBFT `/abci_query` response. `value` is base64-encoded protobuf (or null when the queried key is absent). */ +export interface RpcAbciQueryResult { + response: { + code?: number; + log?: string; + value: string | null; + height?: string; + }; } /** CometBFT `/genesis_chunked` response. `chunk`/`total` are marshaled as strings; `data` is base64-encoded genesis JSON. */ diff --git a/apps/chain-indexer/tsup.config.ts b/apps/chain-indexer/tsup.config.ts index fb175af820..9457387734 100644 --- a/apps/chain-indexer/tsup.config.ts +++ b/apps/chain-indexer/tsup.config.ts @@ -13,6 +13,7 @@ export default defineConfig(async overrideOptions => prependEffectsToEntries: ["reflect-metadata", "@akashnetwork/env-loader"], entry: { server: "./src/server.ts", + reconcile: "./src/reconcile/reconcile.ts", instrumentation: fileURLToPath(import.meta.resolve("@akashnetwork/instrumentation/register")) }, target: tsconfig.compilerOptions.target, diff --git a/package-lock.json b/package-lock.json index 7b01409ede..97a6c32042 100644 --- a/package-lock.json +++ b/package-lock.json @@ -541,6 +541,7 @@ "@akashnetwork/instrumentation": "*", "@akashnetwork/logging": "*", "@akashnetwork/net": "*", + "@cosmjs/amino": "~0.38.0", "@cosmjs/encoding": "~0.38.0", "@cosmjs/proto-signing": "~0.38.0", "@cosmjs/stargate": "~0.38.0", @@ -576,6 +577,34 @@ "vitest-mock-extended": "^4.0.0" } }, + "apps/chain-indexer/node_modules/@cosmjs/amino": { + "version": "0.38.1", + "resolved": "https://registry.npmjs.org/@cosmjs/amino/-/amino-0.38.1.tgz", + "integrity": "sha512-WaThDpq2JwUyKuazq08Xa+FHzQ3jh1HcYnGL4xsyfqFwOlAvnl0EDvSSz9WSwz1oopIxFE9Qtf3OUKOlxBZbYA==", + "license": "Apache-2.0", + "dependencies": { + "@cosmjs/crypto": "^0.38.1", + "@cosmjs/encoding": "^0.38.1", + "@cosmjs/math": "^0.38.1", + "@cosmjs/utils": "^0.38.1" + } + }, + "apps/chain-indexer/node_modules/@cosmjs/crypto": { + "version": "0.38.1", + "resolved": "https://registry.npmjs.org/@cosmjs/crypto/-/crypto-0.38.1.tgz", + "integrity": "sha512-r1KQCjKAdMga2aZ/nkgULRF4fisPZMF6ErucVsMmkASBgDl0k9/vD9K9fHUdGClMv0oOYEfwOT/UTBR7K2OuYA==", + "license": "Apache-2.0", + "dependencies": { + "@cosmjs/encoding": "^0.38.1", + "@cosmjs/math": "^0.38.1", + "@cosmjs/utils": "^0.38.1", + "@noble/ciphers": "^1.3.0", + "@noble/curves": "^1.9.2", + "@noble/hashes": "^1.8.0", + "@scure/bip39": "^1.6.0", + "hash-wasm": "^4.12.0" + } + }, "apps/chain-indexer/node_modules/@cosmjs/encoding": { "version": "0.38.1", "resolved": "https://registry.npmjs.org/@cosmjs/encoding/-/encoding-0.38.1.tgz", @@ -587,6 +616,45 @@ "readonly-date-esm": "^2.0.0" } }, + "apps/chain-indexer/node_modules/@cosmjs/math": { + "version": "0.38.1", + "resolved": "https://registry.npmjs.org/@cosmjs/math/-/math-0.38.1.tgz", + "integrity": "sha512-MBk7p6kPNULi0TusD8O3xoBskFIkRzOtpmnea3sXbTVnguX7epNPVDITXM4tlsg8kAQrEOEIA0g5zAJxzH3Ikw==", + "license": "Apache-2.0" + }, + "apps/chain-indexer/node_modules/@cosmjs/utils": { + "version": "0.38.1", + "resolved": "https://registry.npmjs.org/@cosmjs/utils/-/utils-0.38.1.tgz", + "integrity": "sha512-ccQ5in6IvsQ+o/SstUdQH1jCJ2+MkJPZK7A/EYwMAFcjV8vzOgJ97LVy6AT24nwdi0/iVa2nbAG+fitUEsgLcA==", + "license": "Apache-2.0" + }, + "apps/chain-indexer/node_modules/@noble/ciphers": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@noble/ciphers/-/ciphers-1.3.0.tgz", + "integrity": "sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==", + "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "apps/chain-indexer/node_modules/@noble/curves": { + "version": "1.9.7", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.9.7.tgz", + "integrity": "sha512-gbKGcRUYIjA3/zCCNaWDciTMFI0dCkvou3TL8Zmy5Nc7sJ47a0jtOeZoTaMxkuqRo9cRhjOdZJXegxYE5FN/xw==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "1.8.0" + }, + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "apps/chain-indexer/node_modules/@scure/base": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@scure/base/-/base-2.2.0.tgz", From f81dbba6c172dbae2bec71a470c2e83742f6af2a Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:43:33 +0530 Subject: [PATCH 07/14] feat(indexer): cosmos staking, distribution and governance handlers (#3591) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(indexer): snapshot cosmos validators, delegations and unbonding Reconcile the validator set against the chain's own staking queries so it matches at the tip. Delegation shares can't be derived exactly from messages, since the token/share exchange rate moves with every reward and slash, so an authoritative snapshot is the only way to keep them exact. - validators gains bond state (status, jailed, tokens, delegator_shares, unbonding height/time); new unbonding_delegations table; validator_status enum - StakingSnapshotService fetches the full set over abci_query, then upserts each validator's row and fully replaces delegations and unbonding in one transaction, interning delegators first - sync triggers a snapshot only once caught up to the tip, throttled by STAKING_SNAPSHOT_INTERVAL_BLOCKS, and never lets a snapshot failure halt sync Part of CON-810. * feat(indexer): index cosmos governance and sharpen distribution reasons Index governance proposals, votes and deposits per block, and tighten the balance-ledger reason for distribution flows. - proposals/proposal_votes/proposal_deposits tables plus proposal_status and vote_option enums - gov deriver reads MsgSubmitProposal (v1 and v1beta1), MsgVote/MsgVoteWeighted and MsgDeposit, taking each proposal's id from its submit_proposal event and its lifecycle from the EndBlock active_proposal/inactive_proposal events; a vote promotes its proposal into voting_period without regressing a result - GovWriter persists them inside the block transaction, resolving proposer, voter and depositor from the account ids the committer already interned (governance actors are always the message signer) - decoder now keeps the gov events; committer calls the gov writer - classifyReason is direction-aware: a credit from the distribution module is a reward/commission, but a fund-community-pool debit is a plain transfer, not a reward Part of CON-810. * fix(indexer): keep same-block gov deposits and track running total deposit The proposal_deposits primary key (proposal, depositor, height) dropped a second same-block deposit from the same depositor under onConflictDoNothing, and proposals.total_deposit was frozen at the submission-time initial deposit. Sum a depositor's same-block deposits in the deriver so none are lost, and recompute total_deposit from the full deposit history after each write so the running total stays correct across blocks and re-committing a block never double-counts. * fix(indexer): dedupe governance votes before the batch upsert A voter can re-vote on the same proposal within one commit batch, either via two MsgVote in a single block or across blocks in a backfill batch, which cosmos allows during the voting period. Both rows share the (proposal_id, voter_account_id) conflict target, so a single ON CONFLICT DO UPDATE over them fails with Postgres 21000 and aborts the whole block-committer transaction. Collapse votes to the latest by height before the upsert, the way aggregateDeposits already does for deposits. * refactor(indexer): fetch validator stake in bounded concurrent batches The staking snapshot fetched each validator's delegations and unbonding one query at a time, so a large validator set serialized hundreds of paginated abci_query round trips and held up the poll loop's next-block pickup for the snapshot's whole duration. Fetch a validator's delegations and unbonding concurrently and process validators in bounded batches. The queries are read-only and pinned to the snapshot height, and the RPC pool already balances concurrent load across nodes. * fix(indexer): skip governance messages from failed transactions deriveGovChanges processed every decoded message regardless of its transaction's result code. A MsgVote or MsgDeposit in a reverted tx (code != 0) still produced a vote upsert, a voting_period promotion, and a deposit row that permanently inflated Proposals.total_deposit, even though cosmos rolled the whole tx back and emitted no handler events for it. The vote and deposit paths read the message body directly rather than correlating against an event, so they had no signal the tx failed. Skip a transaction's messages when its code is non-zero, matching the event-driven balance deriver that is already correct for failed txs. * fix(indexer): guard the vote upsert against out-of-order commits writeVotes upserted last-writer-wins on (proposalId, voterAccountId) with no height check, so if two overlapping writers (e.g. pods during a rolling deploy, which block-committer documents as tolerated) commit a voter's re-votes out of height order, the older-height commit silently overwrote the newer vote. dedupeVotes only collapses re-votes merged into a single batch. Add a setWhere requiring excluded.height >= proposal_votes.height, mirroring the GREATEST() guard the committer already uses for the checkpoint, so a stale vote can no longer regress a newer one. * fix(indexer): derive validator consensus hex address in the staking snapshot upsertValidators left hex_address to genesis, so a validator created after genesis — whose row only ever comes from the periodic snapshot — kept a null consensus address forever. Derive it from the query's ed25519 consensus pubkey (the same value genesis computes) and include it in the upsert, so every validator the snapshot sees gets its hex_address. * refactor(indexer): sum proposal deposits in linear time refreshTotalDeposits rebuilt each proposal's amount array by spreading the whole accumulated array on every deposit row, which is O(n^2) in the deposit count. Push into the existing array and sum once instead: same result, linear time. * fix(indexer): snapshot staking after the observed tip is committed Live follow on a 6s block time never went strictly ahead of the tip, so the snapshot never ran. Take the snapshot after the observed tip is committed, honor RPC_TIMEOUT_MS for TLS connect, and fetch validator stake two at a time so a single archival node is not flooded. Part of CON-810. * feat(indexer): read large genesis documents from a local file Mainnet genesis over /genesis_chunked from the archival RPC is too slow to be practical. GENESIS_FILE points at a local JSON file; chain_id is still checked against the RPC node. Part of CON-810. * fix(indexer): resolve early-mainnet submit_proposal event pairs Akashnet-2 height 14675 emits two submit_proposal events with no msg_index: one has proposal_id, the other has proposal_type and voting_period_start. The deriver required a single unmatched event and dropped the proposal. Take the first event that carries an id, and promote to voting_period when voting_period_start is present. Part of CON-810. * fix(indexer): retry transient staking snapshot queries A single ECONNRESET from the archival node aborted the whole validator set fetch. Retry each abci_query a few times so a dropped TLS handshake does not discard the rest of the snapshot. Part of CON-810. * fix(indexer): tighten sync, gov and snapshot after review Skip the poll sleep between catch-up rounds so a slow RPC does not fall further behind. Consume used submit_proposal fallback events so two early-mainnet submits in one tx do not share an id. Classify an EndBlock community-pool spend as a transfer. Abort snapshot retries when sync is stopping. Part of CON-810. * fix(indexer): ignore snapshot insert collisions on overlap Two sync pods snapshotting at once can DELETE then INSERT the same delegation keys; the second insert now no-ops instead of aborting the transaction. Specs import the shared rowsFor helper. Part of CON-810. * fix(indexer): keep a stub proposal when the submit body is dropped A MsgSubmitProposal whose body is null (oversized or undecodable) still gets a Proposals row from the submit_proposal event, using the signer as proposer, so later votes and deposits are not orphaned. Part of CON-810. --- apps/chain-indexer/README.md | 2 +- .../drizzle/0003_mean_smasher.sql | 18 + .../drizzle/0004_pretty_mister_sinister.sql | 41 + .../drizzle/meta/0003_snapshot.json | 921 +++++++++++++ .../drizzle/meta/0004_snapshot.json | 1194 +++++++++++++++++ apps/chain-indexer/drizzle/meta/_journal.json | 14 + apps/chain-indexer/env/.env.sample | 1 + apps/chain-indexer/package.json | 1 + .../src/config/env.config.spec.ts | 12 + apps/chain-indexer/src/config/env.config.ts | 13 + apps/chain-indexer/src/db/schema.spec.ts | 38 +- apps/chain-indexer/src/db/schema.ts | 96 +- .../src/genesis/genesis-source.spec.ts | 31 +- .../src/genesis/genesis-source.ts | 51 +- apps/chain-indexer/src/gov/coin-total.ts | 14 + .../chain-indexer/src/gov/gov-deriver.spec.ts | 290 ++++ apps/chain-indexer/src/gov/gov-deriver.ts | 284 ++++ .../src/gov/gov-writer.service.spec.ts | 228 ++++ .../src/gov/gov-writer.service.ts | 158 +++ .../src/pipeline/balance/balance-deriver.ts | 5 +- .../balance/reason-classifier.spec.ts | 72 +- .../src/pipeline/balance/reason-classifier.ts | 40 +- .../pipeline/block-committer.service.spec.ts | 20 +- .../src/pipeline/block-committer.service.ts | 11 +- .../src/pipeline/block-decoder.service.ts | 18 +- .../src/pipeline/sync-runner.service.spec.ts | 77 +- .../src/pipeline/sync-runner.service.ts | 47 +- .../src/rpc/rpc-client-pool.service.spec.ts | 9 + .../src/rpc/rpc-client-pool.service.ts | 9 +- .../src/staking/staking-query.spec.ts | 209 +++ .../src/staking/staking-query.ts | 218 +++ .../staking/staking-snapshot.service.spec.ts | 274 ++++ .../src/staking/staking-snapshot.service.ts | 244 ++++ package-lock.json | 1 + 34 files changed, 4609 insertions(+), 52 deletions(-) create mode 100644 apps/chain-indexer/drizzle/0003_mean_smasher.sql create mode 100644 apps/chain-indexer/drizzle/0004_pretty_mister_sinister.sql create mode 100644 apps/chain-indexer/drizzle/meta/0003_snapshot.json create mode 100644 apps/chain-indexer/drizzle/meta/0004_snapshot.json create mode 100644 apps/chain-indexer/src/gov/coin-total.ts create mode 100644 apps/chain-indexer/src/gov/gov-deriver.spec.ts create mode 100644 apps/chain-indexer/src/gov/gov-deriver.ts create mode 100644 apps/chain-indexer/src/gov/gov-writer.service.spec.ts create mode 100644 apps/chain-indexer/src/gov/gov-writer.service.ts create mode 100644 apps/chain-indexer/src/staking/staking-query.spec.ts create mode 100644 apps/chain-indexer/src/staking/staking-query.ts create mode 100644 apps/chain-indexer/src/staking/staking-snapshot.service.spec.ts create mode 100644 apps/chain-indexer/src/staking/staking-snapshot.service.ts diff --git a/apps/chain-indexer/README.md b/apps/chain-indexer/README.md index 0f034b3acc..8f41264b2d 100644 --- a/apps/chain-indexer/README.md +++ b/apps/chain-indexer/README.md @@ -44,7 +44,7 @@ The checkpoint height should advance as blocks land in `cosmos.blocks`, `cosmos. Set `GENESIS_IMPORT=true` to seed genesis state before the first block: accounts, per-denom balances (as `genesis`-reason ledger entries), validators, and staking delegations, all in one transaction. Because balance history is only trustworthy from the network's genesis, a fresh `sync` with the flag on must begin at the genesis height, and a fresh start anywhere else is rejected with a clear error. On sandbox that height is 1 (`SYNC_START_HEIGHT=1`); the height is read from the genesis file itself, so a chain continued from an export uses its continuation height. -The import runs once. A `genesis` checkpoint in `indexer_state` makes a restart skip it, and the seed commits in a single transaction, so an interrupted run rolls back and retries cleanly. Genesis is fetched over RPC `/genesis_chunked` from the same nodes sync uses, and its `chain_id` must match the chain being indexed. Leave the flag unset (the default) and sync tails blocks, transactions, and messages from any height exactly as before. +The import runs once. A `genesis` checkpoint in `indexer_state` makes a restart skip it, and the seed commits in a single transaction, so an interrupted run rolls back and retries cleanly. Genesis is fetched over RPC `/genesis_chunked` from the same nodes sync uses, unless `GENESIS_FILE` points at a local JSON file (the practical path for a large mainnet genesis). Either way its `chain_id` must match the chain being indexed. Leave the flag unset (the default) and sync tails blocks, transactions, and messages from any height exactly as before. ## Backfill diff --git a/apps/chain-indexer/drizzle/0003_mean_smasher.sql b/apps/chain-indexer/drizzle/0003_mean_smasher.sql new file mode 100644 index 0000000000..4c88209238 --- /dev/null +++ b/apps/chain-indexer/drizzle/0003_mean_smasher.sql @@ -0,0 +1,18 @@ +CREATE TYPE "cosmos"."validator_status" AS ENUM('unbonded', 'unbonding', 'bonded');--> statement-breakpoint +CREATE TABLE "cosmos"."unbonding_delegations" ( + "delegator_account_id" integer NOT NULL, + "validator_operator_address" text NOT NULL, + "creation_height" bigint NOT NULL, + "completion_time" timestamp with time zone NOT NULL, + "initial_balance" numeric(38, 0) NOT NULL, + "balance" numeric(38, 0) NOT NULL, + CONSTRAINT "unbonding_delegations_delegator_account_id_validator_operator_address_creation_height_pk" PRIMARY KEY("delegator_account_id","validator_operator_address","creation_height") +); +--> statement-breakpoint +ALTER TABLE "cosmos"."validators" ADD COLUMN "jailed" boolean DEFAULT false NOT NULL;--> statement-breakpoint +ALTER TABLE "cosmos"."validators" ADD COLUMN "status" "cosmos"."validator_status";--> statement-breakpoint +ALTER TABLE "cosmos"."validators" ADD COLUMN "tokens" numeric(38, 0);--> statement-breakpoint +ALTER TABLE "cosmos"."validators" ADD COLUMN "delegator_shares" numeric(38, 18);--> statement-breakpoint +ALTER TABLE "cosmos"."validators" ADD COLUMN "unbonding_height" bigint;--> statement-breakpoint +ALTER TABLE "cosmos"."validators" ADD COLUMN "unbonding_time" timestamp with time zone;--> statement-breakpoint +ALTER TABLE "cosmos"."unbonding_delegations" ADD CONSTRAINT "unbonding_delegations_delegator_account_id_accounts_id_fk" FOREIGN KEY ("delegator_account_id") REFERENCES "cosmos"."accounts"("id") ON DELETE no action ON UPDATE no action; \ No newline at end of file diff --git a/apps/chain-indexer/drizzle/0004_pretty_mister_sinister.sql b/apps/chain-indexer/drizzle/0004_pretty_mister_sinister.sql new file mode 100644 index 0000000000..b88a855c87 --- /dev/null +++ b/apps/chain-indexer/drizzle/0004_pretty_mister_sinister.sql @@ -0,0 +1,41 @@ +CREATE TYPE "cosmos"."proposal_status" AS ENUM('deposit_period', 'voting_period', 'passed', 'rejected', 'failed');--> statement-breakpoint +CREATE TYPE "cosmos"."vote_option" AS ENUM('yes', 'abstain', 'no', 'no_with_veto');--> statement-breakpoint +CREATE TABLE "cosmos"."proposal_deposits" ( + "proposal_id" bigint NOT NULL, + "depositor_account_id" integer NOT NULL, + "amount" jsonb NOT NULL, + "height" bigint NOT NULL, + CONSTRAINT "proposal_deposits_proposal_id_depositor_account_id_height_pk" PRIMARY KEY("proposal_id","depositor_account_id","height") +); +--> statement-breakpoint +CREATE TABLE "cosmos"."proposal_votes" ( + "proposal_id" bigint NOT NULL, + "voter_account_id" integer NOT NULL, + "options" jsonb NOT NULL, + "height" bigint NOT NULL, + CONSTRAINT "proposal_votes_proposal_id_voter_account_id_pk" PRIMARY KEY("proposal_id","voter_account_id") +); +--> statement-breakpoint +CREATE TABLE "cosmos"."proposals" ( + "id" bigint PRIMARY KEY NOT NULL, + "proposer_account_id" integer, + "title" text, + "summary" text, + "messages" jsonb, + "metadata" text, + "status" "cosmos"."proposal_status" NOT NULL, + "submit_time" timestamp with time zone, + "deposit_end_time" timestamp with time zone, + "voting_start_time" timestamp with time zone, + "voting_end_time" timestamp with time zone, + "total_deposit" jsonb, + "final_tally_yes" numeric(38, 0), + "final_tally_abstain" numeric(38, 0), + "final_tally_no" numeric(38, 0), + "final_tally_no_with_veto" numeric(38, 0), + "submit_height" bigint NOT NULL +); +--> statement-breakpoint +ALTER TABLE "cosmos"."proposal_deposits" ADD CONSTRAINT "proposal_deposits_depositor_account_id_accounts_id_fk" FOREIGN KEY ("depositor_account_id") REFERENCES "cosmos"."accounts"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "cosmos"."proposal_votes" ADD CONSTRAINT "proposal_votes_voter_account_id_accounts_id_fk" FOREIGN KEY ("voter_account_id") REFERENCES "cosmos"."accounts"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "cosmos"."proposals" ADD CONSTRAINT "proposals_proposer_account_id_accounts_id_fk" FOREIGN KEY ("proposer_account_id") REFERENCES "cosmos"."accounts"("id") ON DELETE no action ON UPDATE no action; \ No newline at end of file diff --git a/apps/chain-indexer/drizzle/meta/0003_snapshot.json b/apps/chain-indexer/drizzle/meta/0003_snapshot.json new file mode 100644 index 0000000000..dbfad5d2cd --- /dev/null +++ b/apps/chain-indexer/drizzle/meta/0003_snapshot.json @@ -0,0 +1,921 @@ +{ + "id": "3dfcc843-b5bb-4265-8203-9317bb7005b6", + "prevId": "b507e1c8-31e3-46c8-8dc2-c0c6927e972d", + "version": "7", + "dialect": "postgresql", + "tables": { + "cosmos.account_balances": { + "name": "account_balances", + "schema": "cosmos", + "columns": { + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "amount": { + "name": "amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "account_balances_account_id_accounts_id_fk": { + "name": "account_balances_account_id_accounts_id_fk", + "tableFrom": "account_balances", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_balances_account_id_denom_pk": { + "name": "account_balances_account_id_denom_pk", + "columns": [ + "account_id", + "denom" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.account_txs": { + "name": "account_txs", + "schema": "cosmos", + "columns": { + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "account_tx_role", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "account_txs_account_id_accounts_id_fk": { + "name": "account_txs_account_id_accounts_id_fk", + "tableFrom": "account_txs", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_txs_account_id_height_tx_index_role_pk": { + "name": "account_txs_account_id_height_tx_index_role_pk", + "columns": [ + "account_id", + "height", + "tx_index", + "role" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.accounts": { + "name": "accounts", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_number": { + "name": "account_number", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "account_type": { + "name": "account_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_module_account": { + "name": "is_module_account", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": { + "accounts_address_idx": { + "name": "accounts_address_idx", + "columns": [ + { + "expression": "address", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.balance_changes": { + "name": "balance_changes", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delta": { + "name": "delta", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "balance_after": { + "name": "balance_after", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "balance_change_reason", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "event_index": { + "name": "event_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "counterparty_account_id": { + "name": "counterparty_account_id", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "balance_changes_account_denom_height_idx": { + "name": "balance_changes_account_denom_height_idx", + "columns": [ + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "denom", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "balance_changes_height_event_index_idx": { + "name": "balance_changes_height_event_index_idx", + "columns": [ + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_index", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "balance_changes_account_id_accounts_id_fk": { + "name": "balance_changes_account_id_accounts_id_fk", + "tableFrom": "balance_changes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "balance_changes_counterparty_account_id_accounts_id_fk": { + "name": "balance_changes_counterparty_account_id_accounts_id_fk", + "tableFrom": "balance_changes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "counterparty_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.blocks": { + "name": "blocks", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "datetime": { + "name": "datetime", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "parent_hash": { + "name": "parent_hash", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "proposer_address": { + "name": "proposer_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tx_count": { + "name": "tx_count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.delegations": { + "name": "delegations", + "schema": "cosmos", + "columns": { + "delegator_account_id": { + "name": "delegator_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "validator_operator_address": { + "name": "validator_operator_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "shares": { + "name": "shares", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "delegations_delegator_account_id_accounts_id_fk": { + "name": "delegations_delegator_account_id_accounts_id_fk", + "tableFrom": "delegations", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "delegator_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "delegations_delegator_account_id_validator_operator_address_pk": { + "name": "delegations_delegator_account_id_validator_operator_address_pk", + "columns": [ + "delegator_account_id", + "validator_operator_address" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.indexer_state": { + "name": "indexer_state", + "schema": "", + "columns": { + "stream": { + "name": "stream", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "last_height": { + "name": "last_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.message_types": { + "name": "message_types", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "message_types_type_idx": { + "name": "message_types_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.messages": { + "name": "messages", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type_id": { + "name": "type_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "body": { + "name": "body", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "messages_type_id_idx": { + "name": "messages_type_id_idx", + "columns": [ + { + "expression": "type_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "messages_type_id_message_types_id_fk": { + "name": "messages_type_id_message_types_id_fk", + "tableFrom": "messages", + "tableTo": "message_types", + "schemaTo": "cosmos", + "columnsFrom": [ + "type_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messages_height_tx_index_index_pk": { + "name": "messages_height_tx_index_index_pk", + "columns": [ + "height", + "tx_index", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.transactions": { + "name": "transactions", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "code": { + "name": "code", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "gas_used": { + "name": "gas_used", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gas_wanted": { + "name": "gas_wanted", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "fee": { + "name": "fee", + "type": "jsonb", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "transactions_hash_idx": { + "name": "transactions_hash_idx", + "columns": [ + { + "expression": "hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "transactions_height_index_pk": { + "name": "transactions_height_index_pk", + "columns": [ + "height", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.unbonding_delegations": { + "name": "unbonding_delegations", + "schema": "cosmos", + "columns": { + "delegator_account_id": { + "name": "delegator_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "validator_operator_address": { + "name": "validator_operator_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "creation_height": { + "name": "creation_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "completion_time": { + "name": "completion_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "initial_balance": { + "name": "initial_balance", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "balance": { + "name": "balance", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "unbonding_delegations_delegator_account_id_accounts_id_fk": { + "name": "unbonding_delegations_delegator_account_id_accounts_id_fk", + "tableFrom": "unbonding_delegations", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "delegator_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "unbonding_delegations_delegator_account_id_validator_operator_address_creation_height_pk": { + "name": "unbonding_delegations_delegator_account_id_validator_operator_address_creation_height_pk", + "columns": [ + "delegator_account_id", + "validator_operator_address", + "creation_height" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.validators": { + "name": "validators", + "schema": "cosmos", + "columns": { + "operator_address": { + "name": "operator_address", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_address": { + "name": "account_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "hex_address": { + "name": "hex_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "moniker": { + "name": "moniker", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "identity": { + "name": "identity", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "details": { + "name": "details", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "security_contact": { + "name": "security_contact", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commission_rate": { + "name": "commission_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "commission_max_rate": { + "name": "commission_max_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "commission_max_change_rate": { + "name": "commission_max_change_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "min_self_delegation": { + "name": "min_self_delegation", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "jailed": { + "name": "jailed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "status": { + "name": "status", + "type": "validator_status", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": false + }, + "tokens": { + "name": "tokens", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "delegator_shares": { + "name": "delegator_shares", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": false + }, + "unbonding_height": { + "name": "unbonding_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "unbonding_time": { + "name": "unbonding_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "cosmos.account_tx_role": { + "name": "account_tx_role", + "schema": "cosmos", + "values": [ + "signer", + "sender", + "receiver" + ] + }, + "cosmos.balance_change_reason": { + "name": "balance_change_reason", + "schema": "cosmos", + "values": [ + "genesis", + "transfer", + "fee", + "reward", + "commission", + "slash", + "gov", + "ibc", + "escrow", + "bme", + "mint", + "burn", + "staking" + ] + }, + "cosmos.validator_status": { + "name": "validator_status", + "schema": "cosmos", + "values": [ + "unbonded", + "unbonding", + "bonded" + ] + } + }, + "schemas": { + "cosmos": "cosmos" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/chain-indexer/drizzle/meta/0004_snapshot.json b/apps/chain-indexer/drizzle/meta/0004_snapshot.json new file mode 100644 index 0000000000..9f6b39314c --- /dev/null +++ b/apps/chain-indexer/drizzle/meta/0004_snapshot.json @@ -0,0 +1,1194 @@ +{ + "id": "84b7320f-f0b3-4a20-ae17-b3bcef49c056", + "prevId": "3dfcc843-b5bb-4265-8203-9317bb7005b6", + "version": "7", + "dialect": "postgresql", + "tables": { + "cosmos.account_balances": { + "name": "account_balances", + "schema": "cosmos", + "columns": { + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "amount": { + "name": "amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "account_balances_account_id_accounts_id_fk": { + "name": "account_balances_account_id_accounts_id_fk", + "tableFrom": "account_balances", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_balances_account_id_denom_pk": { + "name": "account_balances_account_id_denom_pk", + "columns": [ + "account_id", + "denom" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.account_txs": { + "name": "account_txs", + "schema": "cosmos", + "columns": { + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "account_tx_role", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "account_txs_account_id_accounts_id_fk": { + "name": "account_txs_account_id_accounts_id_fk", + "tableFrom": "account_txs", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_txs_account_id_height_tx_index_role_pk": { + "name": "account_txs_account_id_height_tx_index_role_pk", + "columns": [ + "account_id", + "height", + "tx_index", + "role" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.accounts": { + "name": "accounts", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_number": { + "name": "account_number", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "account_type": { + "name": "account_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_module_account": { + "name": "is_module_account", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": { + "accounts_address_idx": { + "name": "accounts_address_idx", + "columns": [ + { + "expression": "address", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.balance_changes": { + "name": "balance_changes", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delta": { + "name": "delta", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "balance_after": { + "name": "balance_after", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "balance_change_reason", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "event_index": { + "name": "event_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "counterparty_account_id": { + "name": "counterparty_account_id", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "balance_changes_account_denom_height_idx": { + "name": "balance_changes_account_denom_height_idx", + "columns": [ + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "denom", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "balance_changes_height_event_index_idx": { + "name": "balance_changes_height_event_index_idx", + "columns": [ + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_index", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "balance_changes_account_id_accounts_id_fk": { + "name": "balance_changes_account_id_accounts_id_fk", + "tableFrom": "balance_changes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "balance_changes_counterparty_account_id_accounts_id_fk": { + "name": "balance_changes_counterparty_account_id_accounts_id_fk", + "tableFrom": "balance_changes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "counterparty_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.blocks": { + "name": "blocks", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "datetime": { + "name": "datetime", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "parent_hash": { + "name": "parent_hash", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "proposer_address": { + "name": "proposer_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tx_count": { + "name": "tx_count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.delegations": { + "name": "delegations", + "schema": "cosmos", + "columns": { + "delegator_account_id": { + "name": "delegator_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "validator_operator_address": { + "name": "validator_operator_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "shares": { + "name": "shares", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "delegations_delegator_account_id_accounts_id_fk": { + "name": "delegations_delegator_account_id_accounts_id_fk", + "tableFrom": "delegations", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "delegator_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "delegations_delegator_account_id_validator_operator_address_pk": { + "name": "delegations_delegator_account_id_validator_operator_address_pk", + "columns": [ + "delegator_account_id", + "validator_operator_address" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.indexer_state": { + "name": "indexer_state", + "schema": "", + "columns": { + "stream": { + "name": "stream", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "last_height": { + "name": "last_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.message_types": { + "name": "message_types", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "message_types_type_idx": { + "name": "message_types_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.messages": { + "name": "messages", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type_id": { + "name": "type_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "body": { + "name": "body", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "messages_type_id_idx": { + "name": "messages_type_id_idx", + "columns": [ + { + "expression": "type_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "messages_type_id_message_types_id_fk": { + "name": "messages_type_id_message_types_id_fk", + "tableFrom": "messages", + "tableTo": "message_types", + "schemaTo": "cosmos", + "columnsFrom": [ + "type_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messages_height_tx_index_index_pk": { + "name": "messages_height_tx_index_index_pk", + "columns": [ + "height", + "tx_index", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.proposal_deposits": { + "name": "proposal_deposits", + "schema": "cosmos", + "columns": { + "proposal_id": { + "name": "proposal_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "depositor_account_id": { + "name": "depositor_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "amount": { + "name": "amount", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "proposal_deposits_depositor_account_id_accounts_id_fk": { + "name": "proposal_deposits_depositor_account_id_accounts_id_fk", + "tableFrom": "proposal_deposits", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "depositor_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "proposal_deposits_proposal_id_depositor_account_id_height_pk": { + "name": "proposal_deposits_proposal_id_depositor_account_id_height_pk", + "columns": [ + "proposal_id", + "depositor_account_id", + "height" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.proposal_votes": { + "name": "proposal_votes", + "schema": "cosmos", + "columns": { + "proposal_id": { + "name": "proposal_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "voter_account_id": { + "name": "voter_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "options": { + "name": "options", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "proposal_votes_voter_account_id_accounts_id_fk": { + "name": "proposal_votes_voter_account_id_accounts_id_fk", + "tableFrom": "proposal_votes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "voter_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "proposal_votes_proposal_id_voter_account_id_pk": { + "name": "proposal_votes_proposal_id_voter_account_id_pk", + "columns": [ + "proposal_id", + "voter_account_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.proposals": { + "name": "proposals", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "proposer_account_id": { + "name": "proposer_account_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "messages": { + "name": "messages", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "proposal_status", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + }, + "submit_time": { + "name": "submit_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deposit_end_time": { + "name": "deposit_end_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "voting_start_time": { + "name": "voting_start_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "voting_end_time": { + "name": "voting_end_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "total_deposit": { + "name": "total_deposit", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "final_tally_yes": { + "name": "final_tally_yes", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "final_tally_abstain": { + "name": "final_tally_abstain", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "final_tally_no": { + "name": "final_tally_no", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "final_tally_no_with_veto": { + "name": "final_tally_no_with_veto", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "submit_height": { + "name": "submit_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "proposals_proposer_account_id_accounts_id_fk": { + "name": "proposals_proposer_account_id_accounts_id_fk", + "tableFrom": "proposals", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "proposer_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.transactions": { + "name": "transactions", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "code": { + "name": "code", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "gas_used": { + "name": "gas_used", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gas_wanted": { + "name": "gas_wanted", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "fee": { + "name": "fee", + "type": "jsonb", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "transactions_hash_idx": { + "name": "transactions_hash_idx", + "columns": [ + { + "expression": "hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "transactions_height_index_pk": { + "name": "transactions_height_index_pk", + "columns": [ + "height", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.unbonding_delegations": { + "name": "unbonding_delegations", + "schema": "cosmos", + "columns": { + "delegator_account_id": { + "name": "delegator_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "validator_operator_address": { + "name": "validator_operator_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "creation_height": { + "name": "creation_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "completion_time": { + "name": "completion_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "initial_balance": { + "name": "initial_balance", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "balance": { + "name": "balance", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "unbonding_delegations_delegator_account_id_accounts_id_fk": { + "name": "unbonding_delegations_delegator_account_id_accounts_id_fk", + "tableFrom": "unbonding_delegations", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "delegator_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "unbonding_delegations_delegator_account_id_validator_operator_address_creation_height_pk": { + "name": "unbonding_delegations_delegator_account_id_validator_operator_address_creation_height_pk", + "columns": [ + "delegator_account_id", + "validator_operator_address", + "creation_height" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.validators": { + "name": "validators", + "schema": "cosmos", + "columns": { + "operator_address": { + "name": "operator_address", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_address": { + "name": "account_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "hex_address": { + "name": "hex_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "moniker": { + "name": "moniker", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "identity": { + "name": "identity", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "details": { + "name": "details", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "security_contact": { + "name": "security_contact", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commission_rate": { + "name": "commission_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "commission_max_rate": { + "name": "commission_max_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "commission_max_change_rate": { + "name": "commission_max_change_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "min_self_delegation": { + "name": "min_self_delegation", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "jailed": { + "name": "jailed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "status": { + "name": "status", + "type": "validator_status", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": false + }, + "tokens": { + "name": "tokens", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "delegator_shares": { + "name": "delegator_shares", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": false + }, + "unbonding_height": { + "name": "unbonding_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "unbonding_time": { + "name": "unbonding_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "cosmos.account_tx_role": { + "name": "account_tx_role", + "schema": "cosmos", + "values": [ + "signer", + "sender", + "receiver" + ] + }, + "cosmos.balance_change_reason": { + "name": "balance_change_reason", + "schema": "cosmos", + "values": [ + "genesis", + "transfer", + "fee", + "reward", + "commission", + "slash", + "gov", + "ibc", + "escrow", + "bme", + "mint", + "burn", + "staking" + ] + }, + "cosmos.proposal_status": { + "name": "proposal_status", + "schema": "cosmos", + "values": [ + "deposit_period", + "voting_period", + "passed", + "rejected", + "failed" + ] + }, + "cosmos.validator_status": { + "name": "validator_status", + "schema": "cosmos", + "values": [ + "unbonded", + "unbonding", + "bonded" + ] + }, + "cosmos.vote_option": { + "name": "vote_option", + "schema": "cosmos", + "values": [ + "yes", + "abstain", + "no", + "no_with_veto" + ] + } + }, + "schemas": { + "cosmos": "cosmos" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/chain-indexer/drizzle/meta/_journal.json b/apps/chain-indexer/drizzle/meta/_journal.json index 43ea741cff..c37ebe033e 100644 --- a/apps/chain-indexer/drizzle/meta/_journal.json +++ b/apps/chain-indexer/drizzle/meta/_journal.json @@ -22,6 +22,20 @@ "when": 1786566671632, "tag": "0002_clever_bulldozer", "breakpoints": true + }, + { + "idx": 3, + "version": "7", + "when": 1786600167399, + "tag": "0003_mean_smasher", + "breakpoints": true + }, + { + "idx": 4, + "version": "7", + "when": 1786601805537, + "tag": "0004_pretty_mister_sinister", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/chain-indexer/env/.env.sample b/apps/chain-indexer/env/.env.sample index 2c48dac39e..2e2768a610 100644 --- a/apps/chain-indexer/env/.env.sample +++ b/apps/chain-indexer/env/.env.sample @@ -5,6 +5,7 @@ RPC_NODE_ENDPOINTS= SYNC_START_HEIGHT= SYNC_POLL_INTERVAL_MS=3000 GENESIS_IMPORT=false +GENESIS_FILE= BACKFILL_FROM_HEIGHT= BACKFILL_TO_HEIGHT= ARCHIVE_BUCKET= diff --git a/apps/chain-indexer/package.json b/apps/chain-indexer/package.json index 6066df516e..c3e9545c41 100644 --- a/apps/chain-indexer/package.json +++ b/apps/chain-indexer/package.json @@ -41,6 +41,7 @@ "postgres": "^3.4.4", "reflect-metadata": "^0.2.2", "tsyringe": "^4.10.0", + "undici": "^7.22.0", "zod": "3.*" }, "devDependencies": { diff --git a/apps/chain-indexer/src/config/env.config.spec.ts b/apps/chain-indexer/src/config/env.config.spec.ts index edc4274aa7..5edc431e76 100644 --- a/apps/chain-indexer/src/config/env.config.spec.ts +++ b/apps/chain-indexer/src/config/env.config.spec.ts @@ -81,6 +81,18 @@ describe("envSchema", () => { expect(config.BACKFILL_TO_HEIGHT).toBeUndefined(); }); + it("treats an empty GENESIS_FILE as absent", () => { + const config = setup({ GENESIS_FILE: "" }); + + expect(config.GENESIS_FILE).toBeUndefined(); + }); + + it("parses GENESIS_FILE when set", () => { + const config = setup({ GENESIS_FILE: "/tmp/genesis.json" }); + + expect(config.GENESIS_FILE).toBe("/tmp/genesis.json"); + }); + it("treats an empty RECONCILE_SAMPLE_SIZE as absent", () => { const config = setup({ RECONCILE_SAMPLE_SIZE: "" }); diff --git a/apps/chain-indexer/src/config/env.config.ts b/apps/chain-indexer/src/config/env.config.ts index c62d647a5d..47f6679afd 100644 --- a/apps/chain-indexer/src/config/env.config.ts +++ b/apps/chain-indexer/src/config/env.config.ts @@ -14,6 +14,14 @@ const rawEnvSchema = z.object({ /** First height to sync when the database has no checkpoint yet. Defaults to the current chain tip. */ SYNC_START_HEIGHT: z.preprocess(emptyStringAsUndefined, z.number({ coerce: true }).int().positive().optional()), SYNC_POLL_INTERVAL_MS: z.number({ coerce: true }).int().positive().default(3_000), + /** + * Reconciles the validator set, delegations and unbonding against the chain once sync catches up to the tip + * and this many blocks have passed since the last snapshot. Delegation shares can't be derived exactly from + * messages, so this authoritative snapshot is what keeps them matching chain queries. + */ + STAKING_SNAPSHOT_INTERVAL_BLOCKS: z.number({ coerce: true }).int().positive().default(1_000), + /** Enables the periodic staking snapshot. Off leaves validators at their genesis and message-derived state. */ + STAKING_SNAPSHOT_ENABLED: z.preprocess(emptyStringAsUndefined, z.enum(["true", "false"]).default("true")).transform(value => value === "true"), /** * Enables the one-time genesis import (accounts, balances, validators, delegations) before the first block. * When on, a fresh sync must start at the network's genesis height or it is rejected as a mid-chain start. @@ -21,6 +29,11 @@ const rawEnvSchema = z.object({ * which treats the string "false" as true. */ GENESIS_IMPORT: z.preprocess(emptyStringAsUndefined, z.enum(["true", "false"]).default("false")).transform(value => value === "true"), + /** + * Path to a genesis JSON file. When set, the import reads this instead of `/genesis_chunked`, which is + * the practical way to seed a large mainnet genesis. The file's chain_id must still match the RPC node. + */ + GENESIS_FILE: z.preprocess(emptyStringAsUndefined, z.string().optional()), /** First height of the backfill range (inclusive). Required when INDEXER_ROLE is "backfill". */ BACKFILL_FROM_HEIGHT: z.preprocess(emptyStringAsUndefined, z.number({ coerce: true }).int().positive().optional()), /** Last height of the backfill range (inclusive). Required when INDEXER_ROLE is "backfill". */ diff --git a/apps/chain-indexer/src/db/schema.spec.ts b/apps/chain-indexer/src/db/schema.spec.ts index a804cf4e13..a1e50d5063 100644 --- a/apps/chain-indexer/src/db/schema.spec.ts +++ b/apps/chain-indexer/src/db/schema.spec.ts @@ -1,7 +1,7 @@ import { getTableConfig } from "drizzle-orm/pg-core"; import { describe, expect, it } from "vitest"; -import { AccountBalances, Accounts, AccountTxs, BalanceChanges, Delegations, Validators } from "@src/db/schema"; +import { AccountBalances, Accounts, AccountTxs, BalanceChanges, Delegations, ProposalDeposits, Proposals, ProposalVotes, UnbondingDelegations, Validators } from "@src/db/schema"; describe("cosmos genesis schema", () => { it("interns accounts under a unique address index", () => { @@ -59,4 +59,40 @@ describe("cosmos genesis schema", () => { expect(config.name).toBe("validators"); expect(config.columns.map(column => column.name)).toContain("operator_address"); }); + + it("enriches validators with snapshot-sourced bond state", () => { + const config = getTableConfig(Validators); + + expect(config.columns.map(column => column.name)).toEqual(expect.arrayContaining(["jailed", "status", "tokens", "delegator_shares", "unbonding_height", "unbonding_time"])); + }); + + it("keys unbonding delegations by delegator, validator and creation height", () => { + const config = getTableConfig(UnbondingDelegations); + + expect(config.name).toBe("unbonding_delegations"); + expect(config.primaryKeys[0].columns.map(column => column.name)).toEqual(["delegator_account_id", "validator_operator_address", "creation_height"]); + expect(config.foreignKeys).toHaveLength(1); + expect(config.foreignKeys[0].reference().foreignColumns[0].name).toBe("id"); + }); + + it("keys proposals by their on-chain id with an optional proposer foreign key", () => { + const config = getTableConfig(Proposals); + + expect(config.name).toBe("proposals"); + expect(config.columns.find(column => column.name === "id")?.primary).toBe(true); + expect(config.foreignKeys[0].reference().foreignColumns[0].name).toBe("id"); + }); + + it("keys proposal votes by proposal and voter", () => { + const config = getTableConfig(ProposalVotes); + + expect(config.primaryKeys[0].columns.map(column => column.name)).toEqual(["proposal_id", "voter_account_id"]); + expect(config.foreignKeys[0].reference().foreignColumns[0].name).toBe("id"); + }); + + it("keys proposal deposits by proposal, depositor and height", () => { + const config = getTableConfig(ProposalDeposits); + + expect(config.primaryKeys[0].columns.map(column => column.name)).toEqual(["proposal_id", "depositor_account_id", "height"]); + }); }); diff --git a/apps/chain-indexer/src/db/schema.ts b/apps/chain-indexer/src/db/schema.ts index 19523cbb7d..1b87720e2e 100644 --- a/apps/chain-indexer/src/db/schema.ts +++ b/apps/chain-indexer/src/db/schema.ts @@ -146,6 +146,9 @@ export const BalanceChanges = cosmosSchema.table( ] ); +/** Consensus bond status, mirroring cosmos `BondStatus` minus the never-valid `UNSPECIFIED`. Written by the staking snapshot. */ +export const validatorStatus = cosmosSchema.enum("validator_status", ["unbonded", "unbonding", "bonded"]); + export const Validators = cosmosSchema.table("validators", { operatorAddress: text("operator_address").primaryKey(), accountAddress: text("account_address"), @@ -158,7 +161,14 @@ export const Validators = cosmosSchema.table("validators", { commissionRate: numeric("commission_rate", { precision: 20, scale: 18 }), commissionMaxRate: numeric("commission_max_rate", { precision: 20, scale: 18 }), commissionMaxChangeRate: numeric("commission_max_change_rate", { precision: 20, scale: 18 }), - minSelfDelegation: numeric("min_self_delegation", { precision: 38, scale: 0 }) + minSelfDelegation: numeric("min_self_delegation", { precision: 38, scale: 0 }), + /** Bond status, self-bonded stake and shares come from the staking snapshot, not from messages, so genesis-seeded rows carry them as null until the first snapshot. */ + jailed: boolean("jailed").notNull().default(false), + status: validatorStatus("status"), + tokens: numeric("tokens", { precision: 38, scale: 0 }), + delegatorShares: numeric("delegator_shares", { precision: 38, scale: 18 }), + unbondingHeight: bigint("unbonding_height", { mode: "number" }), + unbondingTime: timestamp("unbonding_time", { withTimezone: true }) }); export const Delegations = cosmosSchema.table( @@ -173,6 +183,26 @@ export const Delegations = cosmosSchema.table( t => [primaryKey({ columns: [t.delegatorAccountId, t.validatorOperatorAddress] })] ); +/** + * In-flight undelegations, one row per unbonding entry. `creationHeight` distinguishes a delegator's + * concurrent entries against the same validator, so it completes the primary key. Fully replaced from the + * staking snapshot rather than tracked incrementally, since an entry disappears silently once it matures. + */ +export const UnbondingDelegations = cosmosSchema.table( + "unbonding_delegations", + { + delegatorAccountId: integer("delegator_account_id") + .notNull() + .references(() => Accounts.id), + validatorOperatorAddress: text("validator_operator_address").notNull(), + creationHeight: bigint("creation_height", { mode: "number" }).notNull(), + completionTime: timestamp("completion_time", { withTimezone: true }).notNull(), + initialBalance: numeric("initial_balance", { precision: 38, scale: 0 }).notNull(), + balance: numeric("balance", { precision: 38, scale: 0 }).notNull() + }, + t => [primaryKey({ columns: [t.delegatorAccountId, t.validatorOperatorAddress, t.creationHeight] })] +); + /** How an address participated in a transaction: it signed it, or it was the sender/recipient of a coin movement. */ export const accountTxRole = cosmosSchema.enum("account_tx_role", ["signer", "sender", "receiver"]); @@ -192,3 +222,67 @@ export const AccountTxs = cosmosSchema.table( }, t => [primaryKey({ columns: [t.accountId, t.height, t.txIndex, t.role] })] ); + +/** Proposal lifecycle. `deposit_period` on submission; the EndBlock gov events drive the terminal states. */ +export const proposalStatus = cosmosSchema.enum("proposal_status", ["deposit_period", "voting_period", "passed", "rejected", "failed"]); + +export const voteOption = cosmosSchema.enum("vote_option", ["yes", "abstain", "no", "no_with_veto"]); + +/** One weighted vote option, matching cosmos `WeightedVoteOption` (a plain `MsgVote` is stored as a single weight-1 option). */ +export interface WeightedVoteOption { + option: (typeof voteOption.enumValues)[number]; + weight: string; +} + +/** + * Governance proposals, keyed by their on-chain id (assigned in the `submit_proposal` event, not the message). + * `title`/`summary` are populated for gov v1 proposals; a v1beta1 proposal keeps its legacy content under + * `messages`. Final tally is left null for a later reconcile — the power-weighted result isn't in the events. + */ +export const Proposals = cosmosSchema.table("proposals", { + id: bigint("id", { mode: "number" }).primaryKey(), + proposerAccountId: integer("proposer_account_id").references(() => Accounts.id), + title: text("title"), + summary: text("summary"), + messages: jsonb("messages"), + metadata: text("metadata"), + status: proposalStatus("status").notNull(), + submitTime: timestamp("submit_time", { withTimezone: true }), + depositEndTime: timestamp("deposit_end_time", { withTimezone: true }), + votingStartTime: timestamp("voting_start_time", { withTimezone: true }), + votingEndTime: timestamp("voting_end_time", { withTimezone: true }), + totalDeposit: jsonb("total_deposit").$type(), + finalTallyYes: numeric("final_tally_yes", { precision: 38, scale: 0 }), + finalTallyAbstain: numeric("final_tally_abstain", { precision: 38, scale: 0 }), + finalTallyNo: numeric("final_tally_no", { precision: 38, scale: 0 }), + finalTallyNoWithVeto: numeric("final_tally_no_with_veto", { precision: 38, scale: 0 }), + submitHeight: bigint("submit_height", { mode: "number" }).notNull() +}); + +/** Latest vote per (proposal, voter) — a re-vote overwrites the prior one, mirroring chain state. */ +export const ProposalVotes = cosmosSchema.table( + "proposal_votes", + { + proposalId: bigint("proposal_id", { mode: "number" }).notNull(), + voterAccountId: integer("voter_account_id") + .notNull() + .references(() => Accounts.id), + options: jsonb("options").$type().notNull(), + height: bigint("height", { mode: "number" }).notNull() + }, + t => [primaryKey({ columns: [t.proposalId, t.voterAccountId] })] +); + +/** One row per depositor per block — same-block deposits are summed by the deriver, so re-committing a block is idempotent and no deposit is lost. `Proposals.total_deposit` is the sum of these rows. */ +export const ProposalDeposits = cosmosSchema.table( + "proposal_deposits", + { + proposalId: bigint("proposal_id", { mode: "number" }).notNull(), + depositorAccountId: integer("depositor_account_id") + .notNull() + .references(() => Accounts.id), + amount: jsonb("amount").$type().notNull(), + height: bigint("height", { mode: "number" }).notNull() + }, + t => [primaryKey({ columns: [t.proposalId, t.depositorAccountId, t.height] })] +); diff --git a/apps/chain-indexer/src/genesis/genesis-source.spec.ts b/apps/chain-indexer/src/genesis/genesis-source.spec.ts index 7d5ec12982..cffd3b5b24 100644 --- a/apps/chain-indexer/src/genesis/genesis-source.spec.ts +++ b/apps/chain-indexer/src/genesis/genesis-source.spec.ts @@ -1,12 +1,41 @@ +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { mock } from "vitest-mock-extended"; -import { RpcGenesisSource } from "@src/genesis/genesis-source"; +import type { EnvConfig } from "@src/config/env.config"; +import { FileGenesisSource, RpcGenesisSource } from "@src/genesis/genesis-source"; import type { LoggerService } from "@src/providers/logging.provider"; import type { RpcClientPool } from "@src/rpc/rpc-client-pool.service"; import { buildParsedGenesis, buildRawGenesis } from "@test/fakes/genesis-fixtures"; +describe(FileGenesisSource.name, () => { + it("parses a genesis file and asserts the chain-id matches the node", async () => { + const path = join(await mkdtemp(join(tmpdir(), "genesis-")), "genesis.json"); + await writeFile(path, JSON.stringify(buildRawGenesis())); + const { source } = setupFile({ path }); + + await expect(source.fetchGenesis()).resolves.toEqual(buildParsedGenesis()); + }); + + it("rejects when the file's chain-id does not match the node", async () => { + const path = join(await mkdtemp(join(tmpdir(), "genesis-")), "genesis.json"); + await writeFile(path, JSON.stringify(buildRawGenesis())); + const { source } = setupFile({ path, nodeChainId: "othernet" }); + + await expect(source.fetchGenesis()).rejects.toThrow('Genesis chain_id "sandbox-2" does not match the RPC chain-id "othernet"'); + }); + + function setupFile(input: { path: string; nodeChainId?: string }) { + const pool = mock(); + pool.getStatus.mockResolvedValue({ node_info: { network: input.nodeChainId ?? "sandbox-2" }, sync_info: { latest_block_height: "100" } }); + const source = new FileGenesisSource(mock({ GENESIS_FILE: input.path }), pool, mock()); + return { source, pool }; + } +}); + describe(RpcGenesisSource.name, () => { it("reassembles multiple genesis chunks into the parsed document", async () => { const { source, pool } = setup({ chunkCount: 3 }); diff --git a/apps/chain-indexer/src/genesis/genesis-source.ts b/apps/chain-indexer/src/genesis/genesis-source.ts index 2a66f61881..dbf5881db8 100644 --- a/apps/chain-indexer/src/genesis/genesis-source.ts +++ b/apps/chain-indexer/src/genesis/genesis-source.ts @@ -1,19 +1,53 @@ import { fromBase64 } from "@cosmjs/encoding"; +import { readFile } from "node:fs/promises"; import type { InjectionToken } from "tsyringe"; -import { container, inject, singleton } from "tsyringe"; +import { container, inject, instancePerContainerCachingFactory, singleton } from "tsyringe"; +import type { EnvConfig } from "@src/config/env.config"; import type { ParsedGenesis } from "@src/genesis/genesis-schema"; import { parseGenesis } from "@src/genesis/genesis-schema"; +import { APP_CONFIG } from "@src/providers/app-config.provider"; import { LoggerService } from "@src/providers/logging.provider"; import { RpcClientPool } from "@src/rpc/rpc-client-pool.service"; -/** Seam over where the genesis document comes from. Swapping to a GitHub-mirror source (for a large mainnet genesis) is a one-line re-registration. */ +/** Seam over where the genesis document comes from. `GENESIS_FILE` selects the file source; otherwise RPC `/genesis_chunked`. */ export interface GenesisSource { fetchGenesis(): Promise; } export const GENESIS_SOURCE: InjectionToken = Symbol("GENESIS_SOURCE"); +async function assertChainId(genesis: ParsedGenesis, pool: RpcClientPool): Promise { + const chainId = (await pool.getStatus()).node_info.network; + + if (genesis.chainId !== chainId) { + throw new Error(`Genesis chain_id "${genesis.chainId}" does not match the RPC chain-id "${chainId}"`); + } + + return genesis; +} + +@singleton() +export class FileGenesisSource implements GenesisSource { + readonly #path: string; + readonly #pool: RpcClientPool; + readonly #logger: LoggerService; + + constructor(@inject(APP_CONFIG) config: EnvConfig, @inject(RpcClientPool) pool: RpcClientPool, @inject(LoggerService) logger: LoggerService) { + this.#path = config.GENESIS_FILE ?? ""; + this.#pool = pool; + this.#logger = logger; + this.#logger.setContext("GENESIS_SOURCE"); + } + + /** Reads a genesis JSON file (the practical path for a large mainnet genesis) and asserts its chain-id matches the RPC node. */ + async fetchGenesis(): Promise { + const raw = await readFile(this.#path); + this.#logger.info({ event: "GENESIS_FILE_READ", path: this.#path, bytes: raw.byteLength }); + return await assertChainId(parseGenesis(JSON.parse(raw.toString("utf8"))), this.#pool); + } +} + @singleton() export class RpcGenesisSource implements GenesisSource { readonly #pool: RpcClientPool; @@ -27,14 +61,7 @@ export class RpcGenesisSource implements GenesisSource { /** Fetches genesis from the same RPC pool the indexer syncs from and asserts its chain-id matches, so balances can only be seeded for the chain being indexed. */ async fetchGenesis(): Promise { - const genesis = parseGenesis(await this.#fetchRawGenesis()); - const chainId = (await this.#pool.getStatus()).node_info.network; - - if (genesis.chainId !== chainId) { - throw new Error(`Genesis chain_id "${genesis.chainId}" does not match the RPC chain-id "${chainId}"`); - } - - return genesis; + return await assertChainId(parseGenesis(await this.#fetchRawGenesis()), this.#pool); } async #fetchRawGenesis(): Promise { @@ -57,4 +84,6 @@ export class RpcGenesisSource implements GenesisSource { } } -container.register(GENESIS_SOURCE, { useToken: RpcGenesisSource }); +container.register(GENESIS_SOURCE, { + useFactory: instancePerContainerCachingFactory(c => (c.resolve(APP_CONFIG).GENESIS_FILE ? c.resolve(FileGenesisSource) : c.resolve(RpcGenesisSource))) +}); diff --git a/apps/chain-indexer/src/gov/coin-total.ts b/apps/chain-indexer/src/gov/coin-total.ts new file mode 100644 index 0000000000..31397b18fd --- /dev/null +++ b/apps/chain-indexer/src/gov/coin-total.ts @@ -0,0 +1,14 @@ +import type { FeeCoin } from "@src/db/schema"; + +/** Sums coin amounts by denom, dropping zero totals and sorting by denom so re-derivation of the same inputs is deterministic. */ +export function sumFeeCoins(coins: FeeCoin[]): FeeCoin[] { + const totalsByDenom = new Map(); + for (const coin of coins) { + totalsByDenom.set(coin.denom, (totalsByDenom.get(coin.denom) ?? 0n) + BigInt(coin.amount)); + } + + return [...totalsByDenom.entries()] + .filter(([, amount]) => amount !== 0n) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([denom, amount]) => ({ denom, amount: amount.toString() })); +} diff --git a/apps/chain-indexer/src/gov/gov-deriver.spec.ts b/apps/chain-indexer/src/gov/gov-deriver.spec.ts new file mode 100644 index 0000000000..58590ce158 --- /dev/null +++ b/apps/chain-indexer/src/gov/gov-deriver.spec.ts @@ -0,0 +1,290 @@ +import { describe, expect, it } from "vitest"; + +import { deriveGovChanges } from "@src/gov/gov-deriver"; +import type { DecodedBlock, DecodedEvent } from "@src/pipeline/decoded-block"; + +const MSG_SUBMIT_PROPOSAL = "/cosmos.gov.v1.MsgSubmitProposal"; +const MSG_SUBMIT_PROPOSAL_V1BETA1 = "/cosmos.gov.v1beta1.MsgSubmitProposal"; +const MSG_VOTE = "/cosmos.gov.v1.MsgVote"; +const MSG_VOTE_WEIGHTED = "/cosmos.gov.v1.MsgVoteWeighted"; +const MSG_DEPOSIT = "/cosmos.gov.v1.MsgDeposit"; +const SUBMIT_TIME = new Date("2026-08-13T00:00:00Z"); + +describe("deriveGovChanges", () => { + it("derives a v1 proposal with the id from its submit_proposal event, plus the initial deposit", () => { + const changes = deriveGovChanges( + block({ + messages: [ + { + typeUrl: MSG_SUBMIT_PROPOSAL, + body: { + proposer: "akash1prop", + title: "Upgrade", + summary: "Do it", + metadata: "meta", + messages: [{ typeUrl: "/x", value: "AA==" }], + initialDeposit: [{ denom: "uakt", amount: "1000" }] + } + } + ], + txEvents: [event("submit_proposal", { proposal_id: "7" }, 0)] + }) + ); + + expect(changes.proposals).toEqual([ + { + id: 7, + proposerAddress: "akash1prop", + title: "Upgrade", + summary: "Do it", + messages: [{ typeUrl: "/x", value: "AA==" }], + metadata: "meta", + submitTime: SUBMIT_TIME, + submitHeight: 100, + initialDeposit: [{ denom: "uakt", amount: "1000" }] + } + ]); + expect(changes.deposits).toEqual([{ proposalId: 7, depositorAddress: "akash1prop", amount: [{ denom: "uakt", amount: "1000" }], height: 100 }]); + }); + + it("keeps a v1beta1 proposal's legacy content under messages and leaves title/summary null", () => { + const changes = deriveGovChanges( + block({ + messages: [ + { + typeUrl: MSG_SUBMIT_PROPOSAL_V1BETA1, + body: { proposer: "akash1prop", content: { typeUrl: "/cosmos.gov.v1beta1.TextProposal", value: "BB==" }, initialDeposit: [] } + } + ], + txEvents: [event("submit_proposal", { proposal_id: "8" }, 0)] + }) + ); + + expect(changes.proposals[0]).toMatchObject({ id: 8, title: null, summary: null, messages: { typeUrl: "/cosmos.gov.v1beta1.TextProposal", value: "BB==" } }); + expect(changes.deposits).toEqual([]); + }); + + it("assigns distinct ids when one early-mainnet tx submits two proposals without msg_index", () => { + const changes = deriveGovChanges( + block({ + messages: [ + { + typeUrl: MSG_SUBMIT_PROPOSAL_V1BETA1, + body: { proposer: "akash1a", content: { typeUrl: "/cosmos.gov.v1beta1.TextProposal", value: "AA==" }, initialDeposit: [] }, + index: 0 + }, + { + typeUrl: MSG_SUBMIT_PROPOSAL_V1BETA1, + body: { proposer: "akash1b", content: { typeUrl: "/cosmos.gov.v1beta1.TextProposal", value: "BB==" }, initialDeposit: [] }, + index: 1 + } + ], + txEvents: [ + event("submit_proposal", { proposal_id: "4" }), + event("submit_proposal", { proposal_type: "Text", voting_period_start: "4" }), + event("submit_proposal", { proposal_id: "5" }), + event("submit_proposal", { proposal_type: "Text", voting_period_start: "5" }) + ] + }) + ); + + expect(changes.proposals.map(proposal => proposal.id)).toEqual([4, 5]); + }); + + it("takes the proposal id from a pair of unindexed submit_proposal events the way early mainnet emits them", () => { + const changes = deriveGovChanges( + block({ + messages: [ + { + typeUrl: MSG_SUBMIT_PROPOSAL_V1BETA1, + body: { + proposer: "akash1prop", + content: { typeUrl: "/cosmos.params.v1beta1.ParameterChangeProposal", value: "AA==" }, + initialDeposit: [{ denom: "uakt", amount: "1000000000" }] + } + } + ], + txEvents: [event("submit_proposal", { proposal_id: "4" }), event("submit_proposal", { proposal_type: "ParameterChange", voting_period_start: "4" })] + }) + ); + + expect(changes.proposals[0]).toMatchObject({ id: 4, proposerAddress: "akash1prop" }); + expect(changes.deposits).toEqual([{ proposalId: 4, depositorAddress: "akash1prop", amount: [{ denom: "uakt", amount: "1000000000" }], height: 100 }]); + expect(changes.statusUpdates).toEqual([{ proposalId: 4, status: "voting_period", onlyFromDepositPeriod: true }]); + }); + + it("records a stub proposal from the submit event when the decoded body is missing", () => { + const changes = deriveGovChanges( + block({ + messages: [{ typeUrl: MSG_SUBMIT_PROPOSAL, body: null }], + txEvents: [event("submit_proposal", { proposal_id: "42" }, 0)], + signerAddresses: ["akash1prop"] + }) + ); + + expect(changes.proposals).toEqual([ + { + id: 42, + proposerAddress: "akash1prop", + title: null, + summary: null, + messages: null, + metadata: null, + submitTime: SUBMIT_TIME, + submitHeight: 100, + initialDeposit: [] + } + ]); + }); + + it("skips a submit proposal whose id cannot be resolved from an event", () => { + const changes = deriveGovChanges(block({ messages: [{ typeUrl: MSG_SUBMIT_PROPOSAL, body: { proposer: "akash1prop", title: "T" } }], txEvents: [] })); + + expect(changes.proposals).toEqual([]); + }); + + it("links each proposal to its own submit_proposal event by msg_index", () => { + const changes = deriveGovChanges( + block({ + messages: [ + { typeUrl: MSG_SUBMIT_PROPOSAL, body: { proposer: "akash1a", title: "A" }, index: 0 }, + { typeUrl: MSG_SUBMIT_PROPOSAL, body: { proposer: "akash1b", title: "B" }, index: 1 } + ], + txEvents: [event("submit_proposal", { proposal_id: "20" }, 1), event("submit_proposal", { proposal_id: "10" }, 0)] + }) + ); + + expect(changes.proposals.map(proposal => [proposal.id, proposal.title])).toEqual([ + [10, "A"], + [20, "B"] + ]); + }); + + it("derives a plain vote as a single full-weight option and promotes the proposal into voting", () => { + const changes = deriveGovChanges(block({ messages: [{ typeUrl: MSG_VOTE, body: { proposalId: "7", voter: "akash1voter", option: 1 } }] })); + + expect(changes.votes).toEqual([{ proposalId: 7, voterAddress: "akash1voter", options: [{ option: "yes", weight: "1.000000000000000000" }], height: 100 }]); + expect(changes.statusUpdates).toEqual([{ proposalId: 7, status: "voting_period", onlyFromDepositPeriod: true }]); + }); + + it("maps a weighted vote's options", () => { + const changes = deriveGovChanges( + block({ + messages: [ + { + typeUrl: MSG_VOTE_WEIGHTED, + body: { + proposalId: "7", + voter: "akash1voter", + options: [ + { option: 1, weight: "0.700000000000000000" }, + { option: 3, weight: "0.300000000000000000" } + ] + } + } + ] + }) + ); + + expect(changes.votes[0].options).toEqual([ + { option: "yes", weight: "0.700000000000000000" }, + { option: "no", weight: "0.300000000000000000" } + ]); + }); + + it("drops a vote whose option is unspecified", () => { + const changes = deriveGovChanges(block({ messages: [{ typeUrl: MSG_VOTE, body: { proposalId: "7", voter: "akash1voter", option: 0 } }] })); + + expect(changes.votes).toEqual([]); + expect(changes.statusUpdates).toEqual([]); + }); + + it("derives a standalone deposit", () => { + const changes = deriveGovChanges( + block({ messages: [{ typeUrl: MSG_DEPOSIT, body: { proposalId: "7", depositor: "akash1dep", amount: [{ denom: "uakt", amount: "500" }] } }] }) + ); + + expect(changes.deposits).toEqual([{ proposalId: 7, depositorAddress: "akash1dep", amount: [{ denom: "uakt", amount: "500" }], height: 100 }]); + }); + + it("sums a proposer's initial deposit and a same-block deposit into one row instead of dropping the second", () => { + const changes = deriveGovChanges( + block({ + messages: [ + { typeUrl: MSG_SUBMIT_PROPOSAL, body: { proposer: "akash1prop", title: "Upgrade", initialDeposit: [{ denom: "uakt", amount: "1000" }] }, index: 0 }, + { typeUrl: MSG_DEPOSIT, body: { proposalId: "7", depositor: "akash1prop", amount: [{ denom: "uakt", amount: "500" }] }, index: 1 } + ], + txEvents: [event("submit_proposal", { proposal_id: "7" }, 0)] + }) + ); + + expect(changes.deposits).toEqual([{ proposalId: 7, depositorAddress: "akash1prop", amount: [{ denom: "uakt", amount: "1500" }], height: 100 }]); + }); + + it("skips vote and deposit messages from a failed transaction", () => { + const changes = deriveGovChanges( + block({ + code: 5, + messages: [ + { typeUrl: MSG_VOTE, body: { proposalId: "7", voter: "akash1voter", option: 1 } }, + { typeUrl: MSG_DEPOSIT, body: { proposalId: "7", depositor: "akash1dep", amount: [{ denom: "uakt", amount: "500" }] } } + ] + }) + ); + + expect(changes.votes).toEqual([]); + expect(changes.deposits).toEqual([]); + expect(changes.statusUpdates).toEqual([]); + }); + + it("maps an active_proposal result to a terminal status", () => { + const changes = deriveGovChanges(block({ blockEvents: [event("active_proposal", { proposal_id: "7", proposal_result: "proposal_passed" })] })); + + expect(changes.statusUpdates).toEqual([{ proposalId: 7, status: "passed" }]); + }); + + it("maps an inactive_proposal to failed", () => { + const changes = deriveGovChanges(block({ blockEvents: [event("inactive_proposal", { proposal_id: "9" })] })); + + expect(changes.statusUpdates).toEqual([{ proposalId: 9, status: "failed" }]); + }); +}); + +function block(input: { + height?: number; + code?: number; + messages?: { typeUrl: string; body: unknown; index?: number }[]; + txEvents?: DecodedEvent[]; + blockEvents?: DecodedEvent[]; + signerAddresses?: string[]; +}): DecodedBlock { + const messages = input.messages ?? []; + return { + height: input.height ?? 100, + datetime: SUBMIT_TIME, + hash: Buffer.alloc(0), + parentHash: null, + proposerAddress: "P", + transactions: + messages.length > 0 || input.txEvents + ? [ + { + index: 0, + hash: Buffer.alloc(0), + code: input.code ?? 0, + gasUsed: 0, + gasWanted: 0, + fee: [], + messages: messages.map((message, index) => ({ index: message.index ?? index, typeUrl: message.typeUrl, body: message.body })), + events: input.txEvents ?? [], + signerAddresses: input.signerAddresses ?? [] + } + ] + : [], + blockEvents: input.blockEvents ?? [] + }; +} + +function event(type: string, attributes: Record, msgIndex?: number): DecodedEvent { + return msgIndex === undefined ? { type, attributes } : { type, attributes, msgIndex }; +} diff --git a/apps/chain-indexer/src/gov/gov-deriver.ts b/apps/chain-indexer/src/gov/gov-deriver.ts new file mode 100644 index 0000000000..bd1c9fb697 --- /dev/null +++ b/apps/chain-indexer/src/gov/gov-deriver.ts @@ -0,0 +1,284 @@ +import type { FeeCoin, proposalStatus, voteOption, WeightedVoteOption } from "@src/db/schema"; +import { sumFeeCoins } from "@src/gov/coin-total"; +import type { DecodedBlock, DecodedEvent, DecodedMessage, DecodedTransaction } from "@src/pipeline/decoded-block"; + +export type ProposalStatus = (typeof proposalStatus.enumValues)[number]; +type VoteOptionValue = (typeof voteOption.enumValues)[number]; + +const SUBMIT_PROPOSAL = new Set(["/cosmos.gov.v1.MsgSubmitProposal", "/cosmos.gov.v1beta1.MsgSubmitProposal"]); +const VOTE = new Set(["/cosmos.gov.v1.MsgVote", "/cosmos.gov.v1beta1.MsgVote"]); +const VOTE_WEIGHTED = new Set(["/cosmos.gov.v1.MsgVoteWeighted", "/cosmos.gov.v1beta1.MsgVoteWeighted"]); +const DEPOSIT = new Set(["/cosmos.gov.v1.MsgDeposit", "/cosmos.gov.v1beta1.MsgDeposit"]); + +const FULL_WEIGHT = "1.000000000000000000"; + +/** cosmos `VoteOption` enum → the stored option; the unspecified/unknown value has no vote and is dropped. */ +const VOTE_OPTION_BY_NUMBER: Record = { 1: "yes", 2: "abstain", 3: "no", 4: "no_with_veto" }; + +/** cosmos EndBlock `active_proposal` result → terminal status. */ +const STATUS_BY_RESULT: Record = { proposal_passed: "passed", proposal_rejected: "rejected", proposal_failed: "failed" }; + +export interface DerivedProposal { + id: number; + proposerAddress: string | null; + title: string | null; + summary: string | null; + messages: unknown; + metadata: string | null; + submitTime: Date; + submitHeight: number; + initialDeposit: FeeCoin[]; +} + +export interface DerivedVote { + proposalId: number; + voterAddress: string; + options: WeightedVoteOption[]; + height: number; +} + +export interface DerivedDeposit { + proposalId: number; + depositorAddress: string; + amount: FeeCoin[]; + height: number; +} + +/** A status change. `onlyFromDepositPeriod` promotes a proposal into `voting_period` without regressing a terminal state. */ +export interface DerivedStatusUpdate { + proposalId: number; + status: ProposalStatus; + onlyFromDepositPeriod?: boolean; +} + +export interface GovChanges { + proposals: DerivedProposal[]; + votes: DerivedVote[]; + deposits: DerivedDeposit[]; + statusUpdates: DerivedStatusUpdate[]; +} + +/** + * Extracts governance entities from a block's messages and events. Proposal ids come from the `submit_proposal` + * event (the message never carries the assigned id); terminal status comes from the EndBlock `active_proposal` / + * `inactive_proposal` events; a vote promotes its proposal into `voting_period` since votes are only cast then. + * Messages in a failed transaction (`code !== 0`) are skipped, since cosmos rolls back all of its state changes + * and the vote/deposit paths read the message body directly rather than correlating against an emitted event. + * A submit whose decoded body is null (oversized or undecodable) still records a stub row from the event so + * later votes and deposits are not orphaned. + */ +export function deriveGovChanges(block: DecodedBlock): GovChanges { + const changes: GovChanges = { proposals: [], votes: [], deposits: [], statusUpdates: [] }; + + for (const tx of block.transactions) { + if (tx.code !== 0) { + continue; + } + const consumedSubmitEvents = new Set(); + for (const message of tx.messages) { + addMessage(changes, message, tx, block, consumedSubmitEvents); + } + } + + for (const event of block.blockEvents) { + addBlockEvent(changes, event); + } + + changes.deposits = aggregateDeposits(changes.deposits); + + return changes; +} + +/** + * Collapses a depositor's deposits to the same proposal within one block into a single summed row. The block is + * the finest timestamp a deposit carries, and the `(proposal, depositor, height)` key can hold only one row, so + * a proposer's initial deposit landing in the same block as a separate `MsgDeposit` is kept as their combined + * total rather than silently dropped on insert. + */ +function aggregateDeposits(deposits: DerivedDeposit[]): DerivedDeposit[] { + const byKey = new Map(); + for (const deposit of deposits) { + const key = `${deposit.proposalId}:${deposit.depositorAddress}:${deposit.height}`; + const existing = byKey.get(key); + if (existing) { + existing.amount = sumFeeCoins([...existing.amount, ...deposit.amount]); + } else { + byKey.set(key, { ...deposit, amount: sumFeeCoins(deposit.amount) }); + } + } + return [...byKey.values()]; +} + +function addMessage(changes: GovChanges, message: DecodedMessage, tx: DecodedTransaction, block: DecodedBlock, consumedSubmitEvents: Set): void { + if (SUBMIT_PROPOSAL.has(message.typeUrl)) { + addProposal(changes, asRecord(message.body), message.index, tx, block, consumedSubmitEvents); + return; + } + + const body = asRecord(message.body); + if (!body) { + return; + } + + if (VOTE.has(message.typeUrl)) { + const option = mapOption(asNumber(body.option)); + addVote(changes, body, option ? [{ option, weight: FULL_WEIGHT }] : [], block.height); + } else if (VOTE_WEIGHTED.has(message.typeUrl)) { + addVote(changes, body, mapWeightedOptions(body.options), block.height); + } else if (DEPOSIT.has(message.typeUrl)) { + addDeposit(changes, asProposalId(body.proposalId), asString(body.depositor), asCoins(body.amount), block.height); + } +} + +function addProposal( + changes: GovChanges, + body: Record | null, + messageIndex: number, + tx: DecodedTransaction, + block: DecodedBlock, + consumedSubmitEvents: Set +): void { + const id = proposalIdFromEvent(tx, messageIndex, consumedSubmitEvents); + if (id === null) { + return; + } + + const proposer = body ? asString(body.proposer) : tx.signerAddresses[0] ?? null; + const initialDeposit = body ? asCoins(body.initialDeposit) : []; + + changes.proposals.push({ + id, + proposerAddress: proposer, + title: body ? asString(body.title) : null, + summary: body ? asString(body.summary) : null, + messages: body ? body.messages ?? body.content ?? null : null, + metadata: body ? asString(body.metadata) : null, + submitTime: block.datetime, + submitHeight: block.height, + initialDeposit + }); + + if (proposer && initialDeposit.length > 0) { + addDeposit(changes, id, proposer, initialDeposit, block.height); + } + + if (hasVotingPeriodStart(tx, id)) { + changes.statusUpdates.push({ proposalId: id, status: "voting_period", onlyFromDepositPeriod: true }); + } +} + +function addVote(changes: GovChanges, body: Record, options: WeightedVoteOption[], height: number): void { + const proposalId = asProposalId(body.proposalId); + const voter = asString(body.voter); + if (proposalId === null || !voter || options.length === 0) { + return; + } + + changes.votes.push({ proposalId, voterAddress: voter, options, height }); + changes.statusUpdates.push({ proposalId, status: "voting_period", onlyFromDepositPeriod: true }); +} + +function addDeposit(changes: GovChanges, proposalId: number | null, depositor: string | null, amount: FeeCoin[], height: number): void { + if (proposalId === null || !depositor || amount.length === 0) { + return; + } + changes.deposits.push({ proposalId, depositorAddress: depositor, amount, height }); +} + +function addBlockEvent(changes: GovChanges, event: DecodedEvent): void { + const proposalId = asProposalId(event.attributes.proposal_id); + if (proposalId === null) { + return; + } + + if (event.type === "active_proposal") { + changes.statusUpdates.push({ proposalId, status: STATUS_BY_RESULT[event.attributes.proposal_result] ?? "failed" }); + } else if (event.type === "inactive_proposal") { + changes.statusUpdates.push({ proposalId, status: "failed" }); + } +} + +/** + * The `submit_proposal` event carries the assigned id, linked by `msg_index`. Older cosmos (mainnet + * genesis-era) emits two events with no index — one with `proposal_id`, one with `proposal_type` / + * `voting_period_start` — so fall back to the next unused event that actually has an id. + */ +function proposalIdFromEvent(tx: DecodedTransaction, messageIndex: number, consumedSubmitEvents: Set): number | null { + const events = tx.events.filter(event => event.type === "submit_proposal"); + const matched = events.find(candidate => candidate.msgIndex === messageIndex); + if (matched) { + return asProposalId(matched.attributes.proposal_id); + } + + for (const event of events) { + if (consumedSubmitEvents.has(event)) { + continue; + } + const proposalId = asProposalId(event.attributes.proposal_id); + if (proposalId !== null) { + consumedSubmitEvents.add(event); + return proposalId; + } + } + + return null; +} + +function hasVotingPeriodStart(tx: DecodedTransaction, proposalId: number): boolean { + return tx.events.some(event => event.type === "submit_proposal" && asProposalId(event.attributes.voting_period_start) === proposalId); +} + +function mapOption(option: number | null): VoteOptionValue | null { + return option === null ? null : VOTE_OPTION_BY_NUMBER[option] ?? null; +} + +function mapWeightedOptions(raw: unknown): WeightedVoteOption[] { + if (!Array.isArray(raw)) { + return []; + } + return raw + .map(entry => { + const record = asRecord(entry); + const option = record ? mapOption(asNumber(record.option)) : null; + return option ? { option, weight: asString(record?.weight) ?? FULL_WEIGHT } : null; + }) + .filter((entry): entry is WeightedVoteOption => entry !== null); +} + +function asRecord(value: unknown): Record | null { + return typeof value === "object" && value !== null && !Array.isArray(value) ? (value as Record) : null; +} + +function asString(value: unknown): string | null { + return typeof value === "string" && value.length > 0 ? value : null; +} + +function asNumber(value: unknown): number | null { + if (typeof value === "number") { + return value; + } + if (typeof value === "string" && value.length > 0) { + const parsed = Number(value); + return Number.isNaN(parsed) ? null : parsed; + } + return null; +} + +function asProposalId(value: unknown): number | null { + const parsed = asNumber(value); + return parsed !== null && Number.isInteger(parsed) && parsed >= 0 ? parsed : null; +} + +function asCoins(value: unknown): FeeCoin[] { + if (!Array.isArray(value)) { + return []; + } + return value + .map(entry => { + const record = asRecord(entry); + const denom = asString(record?.denom); + const amount = asString(record?.amount); + return denom && amount ? { denom, amount } : null; + }) + .filter((coin): coin is FeeCoin => coin !== null); +} diff --git a/apps/chain-indexer/src/gov/gov-writer.service.spec.ts b/apps/chain-indexer/src/gov/gov-writer.service.spec.ts new file mode 100644 index 0000000000..94576add5e --- /dev/null +++ b/apps/chain-indexer/src/gov/gov-writer.service.spec.ts @@ -0,0 +1,228 @@ +import type { SQL } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { describe, expect, it } from "vitest"; + +import { ProposalDeposits, Proposals, ProposalVotes } from "@src/db/schema"; +import { GovWriter } from "@src/gov/gov-writer.service"; +import type { DecodedBlock, DecodedEvent } from "@src/pipeline/decoded-block"; +import type { ChainTransaction } from "@src/providers/db.provider"; + +import { rowsFor } from "@test/fakes/build-tx-fake"; + +const MSG_SUBMIT_PROPOSAL = "/cosmos.gov.v1.MsgSubmitProposal"; +const MSG_VOTE = "/cosmos.gov.v1.MsgVote"; +const MSG_DEPOSIT = "/cosmos.gov.v1.MsgDeposit"; + +describe(GovWriter.name, () => { + it("inserts a submitted proposal with its proposer id, deposit_period status and initial deposit", async () => { + const { govWriter, tx, inserts } = setup(); + + await govWriter.writeForBlocks( + tx, + [ + block({ + messages: [{ typeUrl: MSG_SUBMIT_PROPOSAL, body: { proposer: "akash1prop", title: "Upgrade", initialDeposit: [{ denom: "uakt", amount: "1000" }] } }], + txEvents: [event("submit_proposal", { proposal_id: "7" }, 0)] + }) + ], + new Map([["akash1prop", 1]]) + ); + + expect(rowsFor(inserts, Proposals)).toEqual([ + expect.objectContaining({ id: 7, proposerAccountId: 1, title: "Upgrade", status: "deposit_period", totalDeposit: [{ denom: "uakt", amount: "1000" }] }) + ]); + expect(rowsFor(inserts, ProposalDeposits)).toEqual([{ proposalId: 7, depositorAccountId: 1, amount: [{ denom: "uakt", amount: "1000" }], height: 100 }]); + }); + + it("refreshes total_deposit from the full deposit history so a later deposit updates the running total", async () => { + const { govWriter, tx, updates } = setup({ priorDeposits: [{ proposalId: 7, amount: [{ denom: "uakt", amount: "1000" }] }] }); + + await govWriter.writeForBlocks( + tx, + [block({ messages: [{ typeUrl: MSG_DEPOSIT, body: { proposalId: "7", depositor: "akash1dep", amount: [{ denom: "uakt", amount: "500" }] } }] })], + new Map([["akash1dep", 3]]) + ); + + const proposalUpdate = updates.find(update => update.table === Proposals); + expect(proposalUpdate?.set).toEqual({ totalDeposit: [{ denom: "uakt", amount: "1500" }] }); + }); + + it("upserts a vote with the resolved voter id and promotes the proposal into voting conditionally", async () => { + const { govWriter, tx, inserts, updates } = setup(); + + await govWriter.writeForBlocks( + tx, + [block({ messages: [{ typeUrl: MSG_VOTE, body: { proposalId: "7", voter: "akash1voter", option: 1 } }] })], + new Map([["akash1voter", 2]]) + ); + + expect(rowsFor(inserts, ProposalVotes)).toEqual([ + { proposalId: 7, voterAccountId: 2, options: [{ option: "yes", weight: "1.000000000000000000" }], height: 100 } + ]); + expect(updates).toHaveLength(1); + expect(updates[0].set).toEqual({ status: "voting_period" }); + expect(whereSql(updates[0].where)).toContain("status"); + }); + + it("collapses a voter's re-vote across blocks in one batch to the latest vote by height", async () => { + const { govWriter, tx, inserts } = setup(); + + await govWriter.writeForBlocks( + tx, + [ + block({ height: 100, messages: [{ typeUrl: MSG_VOTE, body: { proposalId: "7", voter: "akash1voter", option: 1 } }] }), + block({ height: 150, messages: [{ typeUrl: MSG_VOTE, body: { proposalId: "7", voter: "akash1voter", option: 3 } }] }) + ], + new Map([["akash1voter", 2]]) + ); + + expect(rowsFor(inserts, ProposalVotes)).toEqual([ + { proposalId: 7, voterAccountId: 2, options: [{ option: "no", weight: "1.000000000000000000" }], height: 150 } + ]); + }); + + it("collapses two votes from the same voter in one block to the last one", async () => { + const { govWriter, tx, inserts } = setup(); + + await govWriter.writeForBlocks( + tx, + [ + block({ + height: 100, + messages: [ + { typeUrl: MSG_VOTE, body: { proposalId: "7", voter: "akash1voter", option: 1 }, index: 0 }, + { typeUrl: MSG_VOTE, body: { proposalId: "7", voter: "akash1voter", option: 3 }, index: 1 } + ] + }) + ], + new Map([["akash1voter", 2]]) + ); + + expect(rowsFor(inserts, ProposalVotes)).toEqual([ + { proposalId: 7, voterAccountId: 2, options: [{ option: "no", weight: "1.000000000000000000" }], height: 100 } + ]); + }); + + it("guards the vote upsert so a lower-height commit cannot overwrite a newer vote", async () => { + const { govWriter, tx, upserts } = setup(); + + await govWriter.writeForBlocks( + tx, + [block({ messages: [{ typeUrl: MSG_VOTE, body: { proposalId: "7", voter: "akash1voter", option: 1 } }] })], + new Map([["akash1voter", 2]]) + ); + + const voteUpsert = upserts.find(upsert => upsert.table === ProposalVotes) as { table: unknown; config: { setWhere: SQL } }; + expect(whereSql(voteUpsert.config.setWhere)).toContain("height"); + expect(whereSql(voteUpsert.config.setWhere)).toContain(">="); + }); + + it("applies a terminal status from an active_proposal without a status condition", async () => { + const { govWriter, tx, updates } = setup(); + + await govWriter.writeForBlocks( + tx, + [block({ blockEvents: [event("active_proposal", { proposal_id: "7", proposal_result: "proposal_passed" })] })], + new Map() + ); + + expect(updates[0].set).toEqual({ status: "passed" }); + expect(whereSql(updates[0].where)).not.toContain("status"); + }); + + it("skips a vote whose voter was never interned", async () => { + const { govWriter, tx, inserts } = setup(); + + await govWriter.writeForBlocks(tx, [block({ messages: [{ typeUrl: MSG_VOTE, body: { proposalId: "7", voter: "akash1voter", option: 1 } }] })], new Map()); + + expect(rowsFor(inserts, ProposalVotes)).toEqual([]); + }); + + it("writes nothing for a block without governance", async () => { + const { govWriter, tx, inserts, updates } = setup(); + + await govWriter.writeForBlocks(tx, [block({ messages: [{ typeUrl: "/cosmos.bank.v1beta1.MsgSend", body: {} }] })], new Map()); + + expect(inserts).toEqual([]); + expect(updates).toEqual([]); + }); + + function setup(input?: { priorDeposits?: { proposalId: number; amount: { denom: string; amount: string }[] }[] }) { + const inserts: { table: unknown; rows: Record[] }[] = []; + const upserts: { table: unknown; config: Record }[] = []; + const updates: { table: unknown; set: Record; where: unknown }[] = []; + const priorDeposits = input?.priorDeposits ?? []; + + const tx = { + insert: (table: unknown) => ({ + values: (rows: Record | Record[]) => { + inserts.push({ table, rows: Array.isArray(rows) ? rows : [rows] }); + return Object.assign(Promise.resolve(), { + onConflictDoNothing: () => Object.assign(Promise.resolve(), { returning: () => Promise.resolve([]) }), + onConflictDoUpdate: (config: Record) => { + upserts.push({ table, config }); + return Promise.resolve(); + } + }); + } + }), + update: (table: unknown) => ({ + set: (set: Record) => ({ + where: (where: unknown) => { + updates.push({ table, set, where }); + return Promise.resolve(); + } + }) + }), + select: () => ({ + from: () => ({ + where: () => + Promise.resolve([...priorDeposits, ...rowsFor(inserts, ProposalDeposits).map(row => ({ proposalId: row.proposalId, amount: row.amount }))]) + }) + }) + }; + + return { govWriter: new GovWriter(), tx: tx as unknown as ChainTransaction, inserts, upserts, updates }; + } +}); + +function whereSql(where: unknown): string { + return new PgDialect().sqlToQuery(where as SQL).sql; +} + +function block(input: { + height?: number; + messages?: { typeUrl: string; body: unknown; index?: number }[]; + txEvents?: DecodedEvent[]; + blockEvents?: DecodedEvent[]; +}): DecodedBlock { + const messages = input.messages ?? []; + return { + height: input.height ?? 100, + datetime: new Date("2026-08-13T00:00:00Z"), + hash: Buffer.alloc(0), + parentHash: null, + proposerAddress: "P", + transactions: + messages.length > 0 || input.txEvents + ? [ + { + index: 0, + hash: Buffer.alloc(0), + code: 0, + gasUsed: 0, + gasWanted: 0, + fee: [], + messages: messages.map((message, index) => ({ index: message.index ?? index, typeUrl: message.typeUrl, body: message.body })), + events: input.txEvents ?? [], + signerAddresses: [] + } + ] + : [], + blockEvents: input.blockEvents ?? [] + }; +} + +function event(type: string, attributes: Record, msgIndex?: number): DecodedEvent { + return msgIndex === undefined ? { type, attributes } : { type, attributes, msgIndex }; +} diff --git a/apps/chain-indexer/src/gov/gov-writer.service.ts b/apps/chain-indexer/src/gov/gov-writer.service.ts new file mode 100644 index 0000000000..f956a6749d --- /dev/null +++ b/apps/chain-indexer/src/gov/gov-writer.service.ts @@ -0,0 +1,158 @@ +import { and, eq, inArray, sql } from "drizzle-orm"; +import chunk from "lodash/chunk"; +import { singleton } from "tsyringe"; + +import { INSERT_CHUNK_SIZE } from "@src/db/insert-chunk-size"; +import { insertChunked } from "@src/db/insert-chunked"; +import type { FeeCoin } from "@src/db/schema"; +import { ProposalDeposits, Proposals, ProposalVotes } from "@src/db/schema"; +import { sumFeeCoins } from "@src/gov/coin-total"; +import type { DerivedDeposit, DerivedProposal, DerivedStatusUpdate, DerivedVote, GovChanges } from "@src/gov/gov-deriver"; +import { deriveGovChanges } from "@src/gov/gov-deriver"; +import type { DecodedBlock } from "@src/pipeline/decoded-block"; +import type { ChainTransaction } from "@src/providers/db.provider"; + +/** + * Persists governance entities inside the block transaction. Proposal ids and their proposer/voter/depositor + * addresses are already interned by the committer (governance actors are always the message signer), so this + * resolves account ids from the passed map rather than interning again. Proposals are inserted conflict-free + * (their id is assigned once) and their status is advanced only by the separate status updates, so re-committing + * a block never regresses a proposal that has since entered voting or reached a terminal result. + */ +@singleton() +export class GovWriter { + async writeForBlocks(tx: ChainTransaction, blocks: DecodedBlock[], accountIds: Map): Promise { + const changes = merge(blocks.map(deriveGovChanges)); + if (changes.proposals.length === 0 && changes.votes.length === 0 && changes.deposits.length === 0 && changes.statusUpdates.length === 0) { + return; + } + + await this.#writeProposals(tx, changes.proposals, accountIds); + await this.#writeVotes(tx, changes.votes, accountIds); + await this.#writeDeposits(tx, changes.deposits, accountIds); + await this.#applyStatusUpdates(tx, changes.statusUpdates); + } + + async #writeProposals(tx: ChainTransaction, proposals: DerivedProposal[], accountIds: Map): Promise { + const rows = proposals.map(proposal => ({ + id: proposal.id, + proposerAccountId: proposal.proposerAddress ? accountIds.get(proposal.proposerAddress) ?? null : null, + title: proposal.title, + summary: proposal.summary, + messages: proposal.messages, + metadata: proposal.metadata, + status: "deposit_period" as const, + submitTime: proposal.submitTime, + totalDeposit: proposal.initialDeposit.length > 0 ? proposal.initialDeposit : null, + submitHeight: proposal.submitHeight + })); + + await insertChunked(tx, Proposals, rows); + } + + /** + * The vote upsert is last-writer-wins on `(proposalId, voterAccountId)`, so the `setWhere` height guard stops + * an out-of-order commit (e.g. overlapping pods on a rolling deploy) from overwriting a newer vote with a + * stale one; `dedupeVotes` handles the same collision among blocks merged into a single batch. + */ + async #writeVotes(tx: ChainTransaction, votes: DerivedVote[], accountIds: Map): Promise { + const rows = dedupeVotes(votes) + .map(vote => { + const voterAccountId = accountIds.get(vote.voterAddress); + return voterAccountId === undefined ? null : { proposalId: vote.proposalId, voterAccountId, options: vote.options, height: vote.height }; + }) + .filter((row): row is NonNullable => row !== null); + + for (const rowChunk of chunk(rows, INSERT_CHUNK_SIZE)) { + await tx + .insert(ProposalVotes) + .values(rowChunk) + .onConflictDoUpdate({ + target: [ProposalVotes.proposalId, ProposalVotes.voterAccountId], + set: { options: sqlExcluded("options"), height: sqlExcluded("height") }, + setWhere: sql`excluded.height >= ${ProposalVotes.height}` + }); + } + } + + async #writeDeposits(tx: ChainTransaction, deposits: DerivedDeposit[], accountIds: Map): Promise { + const rows = deposits + .map(deposit => { + const depositorAccountId = accountIds.get(deposit.depositorAddress); + return depositorAccountId === undefined ? null : { proposalId: deposit.proposalId, depositorAccountId, amount: deposit.amount, height: deposit.height }; + }) + .filter((row): row is NonNullable => row !== null); + + if (rows.length === 0) { + return; + } + + await insertChunked(tx, ProposalDeposits, rows); + await this.#refreshTotalDeposits(tx, [...new Set(rows.map(row => row.proposalId))]); + } + + /** + * Recomputes each touched proposal's `total_deposit` from its full deposit history rather than incrementing, + * so the running total stays correct across blocks and a re-committed block never double-counts. + */ + async #refreshTotalDeposits(tx: ChainTransaction, proposalIds: number[]): Promise { + const deposits = await tx + .select({ proposalId: ProposalDeposits.proposalId, amount: ProposalDeposits.amount }) + .from(ProposalDeposits) + .where(inArray(ProposalDeposits.proposalId, proposalIds)); + + const amountsByProposal = new Map(); + for (const deposit of deposits) { + const amounts = amountsByProposal.get(deposit.proposalId) ?? []; + amounts.push(...deposit.amount); + amountsByProposal.set(deposit.proposalId, amounts); + } + + for (const proposalId of proposalIds) { + const total = sumFeeCoins(amountsByProposal.get(proposalId) ?? []); + await tx + .update(Proposals) + .set({ totalDeposit: total.length > 0 ? total : null }) + .where(eq(Proposals.id, proposalId)); + } + } + + /** A `voting_period` promotion is conditional so it can't overwrite a terminal status; terminal updates are unconditional. */ + async #applyStatusUpdates(tx: ChainTransaction, updates: DerivedStatusUpdate[]): Promise { + for (const update of updates) { + const filter = update.onlyFromDepositPeriod ? and(eq(Proposals.id, update.proposalId), eq(Proposals.status, "deposit_period")) : eq(Proposals.id, update.proposalId); + await tx.update(Proposals).set({ status: update.status }).where(filter); + } + } +} + +function merge(perBlock: GovChanges[]): GovChanges { + return { + proposals: perBlock.flatMap(changes => changes.proposals), + votes: perBlock.flatMap(changes => changes.votes), + deposits: perBlock.flatMap(changes => changes.deposits), + statusUpdates: perBlock.flatMap(changes => changes.statusUpdates) + }; +} + +/** + * Collapses a voter's re-votes on the same proposal within one commit batch to their latest by height. A batch + * merges votes across many blocks and cosmos allows re-voting during `voting_period`, so the same + * `(proposalId, voterAccountId)` — the vote's primary key — can appear more than once; feeding both to one + * `ON CONFLICT DO UPDATE` would raise `21000: command cannot affect row a second time` and abort the batch. + */ +function dedupeVotes(votes: DerivedVote[]): DerivedVote[] { + const byKey = new Map(); + for (const vote of votes) { + const key = `${vote.proposalId}:${vote.voterAddress}`; + const existing = byKey.get(key); + if (!existing || vote.height >= existing.height) { + byKey.set(key, vote); + } + } + return [...byKey.values()]; +} + +function sqlExcluded(column: string) { + return sql.raw(`excluded.${column}`); +} diff --git a/apps/chain-indexer/src/pipeline/balance/balance-deriver.ts b/apps/chain-indexer/src/pipeline/balance/balance-deriver.ts index 9ebca7c13b..6616343f52 100644 --- a/apps/chain-indexer/src/pipeline/balance/balance-deriver.ts +++ b/apps/chain-indexer/src/pipeline/balance/balance-deriver.ts @@ -130,7 +130,10 @@ export function deriveBalanceChanges(block: DecodedBlock, registry: ModuleAddres for (const coin of parseCoins(event.attributes.amount ?? "")) { const counterpartyAddress = correlateCounterparty(scope, holder, coin.denom, coin.amount, direction); - const reason = classifyReason({ address: holder, counterpartyAddress, denom: coin.denom, isMint, isBurn, isSlash, msgTypeUrl }, registry); + const reason = classifyReason( + { address: holder, counterpartyAddress, denom: coin.denom, isMint, isBurn, isSlash, isCredit: direction === "received", msgTypeUrl }, + registry + ); changes.push({ address: holder, diff --git a/apps/chain-indexer/src/pipeline/balance/reason-classifier.spec.ts b/apps/chain-indexer/src/pipeline/balance/reason-classifier.spec.ts index 39dfc4da60..d76d5ac6a1 100644 --- a/apps/chain-indexer/src/pipeline/balance/reason-classifier.spec.ts +++ b/apps/chain-indexer/src/pipeline/balance/reason-classifier.spec.ts @@ -7,21 +7,29 @@ import { classifyReason } from "@src/pipeline/balance/reason-classifier"; const registry = buildModuleAddressRegistry(AKASH_ADDRESS_PREFIX); const moduleAddress = (name: string) => deriveModuleAddress(name, AKASH_ADDRESS_PREFIX); +const WITHDRAW_DELEGATOR_REWARD = "/cosmos.distribution.v1beta1.MsgWithdrawDelegatorReward"; +const WITHDRAW_VALIDATOR_COMMISSION = "/cosmos.distribution.v1beta1.MsgWithdrawValidatorCommission"; describe("classifyReason", () => { it("classifies a payment to the fee collector as a fee", () => { expect(classifyReason(context({ counterpartyAddress: moduleAddress("fee_collector") }), registry)).toBe("fee"); }); - it("classifies a distribution outflow as a reward", () => { - expect(classifyReason(context({ counterpartyAddress: moduleAddress("distribution") }), registry)).toBe("reward"); + it("classifies a credit from the distribution module as a reward", () => { + expect( + classifyReason(context({ counterpartyAddress: moduleAddress("distribution"), isCredit: true, msgTypeUrl: WITHDRAW_DELEGATOR_REWARD }), registry) + ).toBe("reward"); }); - it("classifies a distribution outflow withdrawn by a validator commission message as commission", () => { - const ctx = context({ counterpartyAddress: moduleAddress("distribution"), msgTypeUrl: "/cosmos.distribution.v1beta1.MsgWithdrawValidatorCommission" }); + it("classifies a credit from distribution withdrawn by a validator commission message as commission", () => { + const ctx = context({ counterpartyAddress: moduleAddress("distribution"), isCredit: true, msgTypeUrl: WITHDRAW_VALIDATOR_COMMISSION }); expect(classifyReason(ctx, registry)).toBe("commission"); }); + it("classifies a debit to the distribution module (fund community pool) as a plain transfer, not a reward", () => { + expect(classifyReason(context({ counterpartyAddress: moduleAddress("distribution"), isCredit: false }), registry)).toBe("transfer"); + }); + it("classifies a flow with the bonded pool as staking", () => { expect(classifyReason(context({ counterpartyAddress: moduleAddress("bonded_tokens_pool") }), registry)).toBe("staking"); }); @@ -74,12 +82,59 @@ describe("classifyReason", () => { expect(classifyReason(context({ counterpartyAddress: "akash1peer" }), registry)).toBe("transfer"); }); - it("tags each leg of a fee_collector-to-distribution movement by the holder's own role", () => { - const feeCollector = moduleAddress("fee_collector"); + it("tags the fee_collector's own leg by its role rather than the distribution counterparty", () => { + expect(classifyReason(context({ address: moduleAddress("fee_collector"), counterpartyAddress: moduleAddress("distribution") }), registry)).toBe("fee"); + }); + + it("tags both legs of a reward withdrawal as reward, including the distribution module's own debit leg", () => { + const distribution = moduleAddress("distribution"); + + expect( + classifyReason( + context({ address: "akash1delegator", counterpartyAddress: distribution, isCredit: true, msgTypeUrl: WITHDRAW_DELEGATOR_REWARD }), + registry + ) + ).toBe("reward"); + expect( + classifyReason( + context({ address: distribution, counterpartyAddress: "akash1delegator", isCredit: false, msgTypeUrl: WITHDRAW_DELEGATOR_REWARD }), + registry + ) + ).toBe("reward"); + }); + + it("tags both legs of a commission withdrawal as commission", () => { + const distribution = moduleAddress("distribution"); + + expect( + classifyReason( + context({ address: "akash1validator", counterpartyAddress: distribution, isCredit: true, msgTypeUrl: WITHDRAW_VALIDATOR_COMMISSION }), + registry + ) + ).toBe("commission"); + expect( + classifyReason( + context({ address: distribution, counterpartyAddress: "akash1validator", isCredit: false, msgTypeUrl: WITHDRAW_VALIDATOR_COMMISSION }), + registry + ) + ).toBe("commission"); + }); + + it("classifies both legs of a community-pool spend as a transfer, not a reward", () => { const distribution = moduleAddress("distribution"); - expect(classifyReason(context({ address: feeCollector, counterpartyAddress: distribution }), registry)).toBe("fee"); - expect(classifyReason(context({ address: distribution, counterpartyAddress: feeCollector }), registry)).toBe("reward"); + expect(classifyReason(context({ address: "akash1recipient", counterpartyAddress: distribution, isCredit: true, msgTypeUrl: null }), registry)).toBe( + "transfer" + ); + expect(classifyReason(context({ address: distribution, counterpartyAddress: "akash1recipient", isCredit: false, msgTypeUrl: null }), registry)).toBe( + "transfer" + ); + }); + + it("does not mistake the distribution module's own funding inflow for a reward", () => { + expect( + classifyReason(context({ address: moduleAddress("distribution"), counterpartyAddress: moduleAddress("fee_collector"), isCredit: true }), registry) + ).toBe("transfer"); }); it("tags the mint module's outgoing forwarding leg as mint rather than the counterparty's fee", () => { @@ -94,6 +149,7 @@ describe("classifyReason", () => { isMint: false, isBurn: false, isSlash: false, + isCredit: false, msgTypeUrl: null, ...overrides }; diff --git a/apps/chain-indexer/src/pipeline/balance/reason-classifier.ts b/apps/chain-indexer/src/pipeline/balance/reason-classifier.ts index 1e1bd748db..ec7b60aa1f 100644 --- a/apps/chain-indexer/src/pipeline/balance/reason-classifier.ts +++ b/apps/chain-indexer/src/pipeline/balance/reason-classifier.ts @@ -3,7 +3,7 @@ import type { ModuleAddressRegistry } from "@src/pipeline/balance/module-address export type BalanceReason = (typeof balanceChangeReason.enumValues)[number]; -/** What the classifier knows about one coin movement: who moved it, the correlated counterparty, and any coincident mint/burn/slash. */ +/** What the classifier knows about one coin movement: who moved it, the correlated counterparty, whether it credits the holder, and any coincident mint/burn/slash. */ export interface ReasonContext { address: string; counterpartyAddress: string | null; @@ -11,18 +11,20 @@ export interface ReasonContext { isMint: boolean; isBurn: boolean; isSlash: boolean; + isCredit: boolean; msgTypeUrl: string | null; } const WITHDRAW_VALIDATOR_COMMISSION = "/cosmos.distribution.v1beta1.MsgWithdrawValidatorCommission"; /** - * MVP reason heuristic. Coincident mint/burn/slash win first (they are unambiguous), then the holder's own - * module role, falling back to the counterparty's, then the denom. Preferring the holder's own role keeps each - * leg of a module-to-module movement (e.g. fee_collector to distribution every block) tagged by the module - * whose balance actually changed, rather than mirroring the counterparty. Anything unrecognized is a plain - * `transfer`. Escrow-module movements classify as `escrow`; per-deployment/lease attribution of that escrow is - * deliberately left for later. + * Reason heuristic. Coincident mint/burn/slash win first (they are unambiguous), then the holder's own module + * role, falling back to the counterparty's, then the denom. Preferring the holder's own role keeps each leg of + * a module-to-module movement (e.g. fee_collector to distribution every block) tagged by the module whose + * balance actually changed, rather than mirroring the counterparty. Distribution flows are direction-aware + * relative to the module so a fund-community-pool inflow or an EndBlock community-pool spend is not mistaken + * for a reward. Anything unrecognized is a plain `transfer`. Escrow-module movements classify as `escrow`; + * per-deployment/lease attribution of that escrow is deliberately left for later. */ export function classifyReason(ctx: ReasonContext, registry: ModuleAddressRegistry): BalanceReason { if (ctx.isSlash) { @@ -35,14 +37,15 @@ export function classifyReason(ctx: ReasonContext, registry: ModuleAddressRegist return "burn"; } - const role = registry.roleOf(ctx.address) ?? (ctx.counterpartyAddress ? registry.roleOf(ctx.counterpartyAddress) : undefined); + const holderRole = registry.roleOf(ctx.address); + const role = holderRole ?? (ctx.counterpartyAddress ? registry.roleOf(ctx.counterpartyAddress) : undefined); switch (role) { case "mint": return "mint"; case "fee_collector": return "fee"; case "distribution": - return ctx.msgTypeUrl === WITHDRAW_VALIDATOR_COMMISSION ? "commission" : "reward"; + return classifyDistributionFlow(ctx, holderRole === "distribution"); case "bonded_tokens_pool": case "not_bonded_tokens_pool": return "staking"; @@ -58,3 +61,22 @@ export function classifyReason(ctx: ReasonContext, registry: ModuleAddressRegist return ctx.denom.startsWith("ibc/") ? "ibc" : "transfer"; } + +/** + * A payout leaving the distribution module is a delegator reward, or commission when the withdraw came from a + * validator's own commission message. Direction is measured against the module itself, not the row's holder: + * whether the module is the holder being debited or the counterparty a holder is credited from, the outflow is + * classified the same on both legs. A flow *into* the module (e.g. MsgFundCommunityPool) is not a reward, so it + * falls back to a plain transfer. An outflow with no signer message is a community-pool spend (EndBlock, no + * msg_index), not a reward withdrawal — those always carry a withdraw message type. + */ +function classifyDistributionFlow(ctx: ReasonContext, distributionIsHolder: boolean): BalanceReason { + const leavesModule = distributionIsHolder ? !ctx.isCredit : ctx.isCredit; + if (!leavesModule) { + return "transfer"; + } + if (ctx.msgTypeUrl === WITHDRAW_VALIDATOR_COMMISSION) { + return "commission"; + } + return ctx.msgTypeUrl === null ? "transfer" : "reward"; +} diff --git a/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts b/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts index 6c542d2268..a88932f20f 100644 --- a/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts +++ b/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts @@ -4,6 +4,7 @@ import { describe, expect, it } from "vitest"; import { mock } from "vitest-mock-extended"; import { AccountTxs, Blocks, IndexerState, Messages, MessageTypes } from "@src/db/schema"; +import type { GovWriter } from "@src/gov/gov-writer.service"; import type { AccountInterner } from "@src/pipeline/balance/account-interner.service"; import type { BalanceWriter } from "@src/pipeline/balance/balance-writer.service"; import { BlockCommitterService } from "@src/pipeline/block-committer.service"; @@ -140,6 +141,20 @@ describe(BlockCommitterService.name, () => { }); }); + describe("governance", () => { + it("hands the governance writer the batch blocks and interned account ids inside the transaction", async () => { + const { committer, insertedRows, govWriter } = setup({ selectResults: [[{ id: 7, type: MSG_SEND }]] }); + + await committer.commit(buildBlock([MSG_SEND], 10, { signerAddresses: ["akash1voter"] })); + + expect(govWriter.writeForBlocks).toHaveBeenCalledWith(expect.anything(), [expect.objectContaining({ height: 10 })], expect.any(Map)); + const accountIds = govWriter.writeForBlocks.mock.calls[0][2]; + expect(accountIds.get("akash1voter")).toBeDefined(); + const order = insertedRows.map(call => call.table); + expect(order.indexOf(AccountTxs)).toBeLessThan(order.indexOf(IndexerState)); + }); + }); + function setup(input?: { selectResults?: Array>; insertReturning?: Array<{ id: number; type: string }> }) { const selectResults = [...(input?.selectResults ?? [[]])]; const insertedRows: Array<{ table: unknown; rows: unknown }> = []; @@ -173,8 +188,9 @@ describe(BlockCommitterService.name, () => { insertedRows.push({ table: BALANCE_WRITE, rows: [] }); }); - const committer = new BlockCommitterService(dbFake as unknown as ChainDatabase, interner, balanceWriter); - return { committer, insertedRows, conflictUpdates, interner, balanceWriter }; + const govWriter = mock(); + const committer = new BlockCommitterService(dbFake as unknown as ChainDatabase, interner, balanceWriter, govWriter); + return { committer, insertedRows, conflictUpdates, interner, balanceWriter, govWriter }; } function buildBlock( diff --git a/apps/chain-indexer/src/pipeline/block-committer.service.ts b/apps/chain-indexer/src/pipeline/block-committer.service.ts index b1e9a97c51..f8ed3879b5 100644 --- a/apps/chain-indexer/src/pipeline/block-committer.service.ts +++ b/apps/chain-indexer/src/pipeline/block-committer.service.ts @@ -3,6 +3,7 @@ import { inject, singleton } from "tsyringe"; import { insertChunked } from "@src/db/insert-chunked"; import { AccountTxs, Blocks, IndexerState, Messages, MessageTypes, Transactions } from "@src/db/schema"; +import { GovWriter } from "@src/gov/gov-writer.service"; import { AccountInterner } from "@src/pipeline/balance/account-interner.service"; import type { DerivedAccountTx } from "@src/pipeline/balance/account-tx-deriver"; import { deriveAccountTxs } from "@src/pipeline/balance/account-tx-deriver"; @@ -22,13 +23,20 @@ export class BlockCommitterService { readonly #db: ChainDatabase; readonly #interner: AccountInterner; readonly #balanceWriter: BalanceWriter; + readonly #govWriter: GovWriter; readonly #moduleRegistry = buildModuleAddressRegistry(); readonly #typeIds = new Map(); - constructor(@inject(CHAIN_DB) db: ChainDatabase, @inject(AccountInterner) interner: AccountInterner, @inject(BalanceWriter) balanceWriter: BalanceWriter) { + constructor( + @inject(CHAIN_DB) db: ChainDatabase, + @inject(AccountInterner) interner: AccountInterner, + @inject(BalanceWriter) balanceWriter: BalanceWriter, + @inject(GovWriter) govWriter: GovWriter + ) { this.#db = db; this.#interner = interner; this.#balanceWriter = balanceWriter; + this.#govWriter = govWriter; } async commit(block: DecodedBlock): Promise { @@ -97,6 +105,7 @@ export class BlockCommitterService { await this.#balanceWriter.write(tx, balanceIntents); await insertChunked(tx, AccountTxs, accountTxRows); + await this.#govWriter.writeForBlocks(tx, blocks, accountIds); await tx .insert(IndexerState) diff --git a/apps/chain-indexer/src/pipeline/block-decoder.service.ts b/apps/chain-indexer/src/pipeline/block-decoder.service.ts index 1c9bed1ff8..faa2070dd0 100644 --- a/apps/chain-indexer/src/pipeline/block-decoder.service.ts +++ b/apps/chain-indexer/src/pipeline/block-decoder.service.ts @@ -13,8 +13,22 @@ import type { Registry } from "@src/providers/type-registry.provider"; import { TYPE_REGISTRY } from "@src/providers/type-registry.provider"; import type { RpcBlockResult, RpcBlockResultsResult, RpcEvent, RpcTxResult } from "@src/rpc/rpc-types"; -/** The ledger derives balances and reasons only from these event types; capturing the rest would waste memory across backfill batches. */ -const RELEVANT_EVENT_TYPES = new Set(["coin_spent", "coin_received", "transfer", "coinbase", "burn", "slash"]); +/** + * The ledger derives balances and reasons from the coin/transfer/mint/burn/slash events; the gov events carry + * proposal ids and lifecycle transitions for the governance handler. Capturing the rest would waste memory + * across backfill batches. + */ +const RELEVANT_EVENT_TYPES = new Set([ + "coin_spent", + "coin_received", + "transfer", + "coinbase", + "burn", + "slash", + "submit_proposal", + "active_proposal", + "inactive_proposal" +]); const MSG_INDEX_ATTRIBUTE = "msg_index"; diff --git a/apps/chain-indexer/src/pipeline/sync-runner.service.spec.ts b/apps/chain-indexer/src/pipeline/sync-runner.service.spec.ts index f998063442..a4748f6c1b 100644 --- a/apps/chain-indexer/src/pipeline/sync-runner.service.spec.ts +++ b/apps/chain-indexer/src/pipeline/sync-runner.service.spec.ts @@ -13,6 +13,7 @@ import type { ChainDatabase } from "@src/providers/db.provider"; import type { LoggerService } from "@src/providers/logging.provider"; import type { RpcClientPool } from "@src/rpc/rpc-client-pool.service"; import type { RpcBlockResult } from "@src/rpc/rpc-types"; +import type { StakingSnapshotService } from "@src/staking/staking-snapshot.service"; describe(SyncRunnerService.name, () => { it("stages every synced block in the archive with its raw payloads", async () => { @@ -135,6 +136,56 @@ describe(SyncRunnerService.name, () => { }); }); + describe("staking snapshot", () => { + it("snapshots at the observed tip after catching up, not only once ahead of the tip", async () => { + const { runner, stakingSnapshot, committer } = setup({ tipHeight: 3, stopOn: "snapshot" }); + + await runner.start(); + + expect(committer.commit).toHaveBeenCalledTimes(3); + expect(stakingSnapshot.snapshot).toHaveBeenCalledTimes(1); + expect(stakingSnapshot.snapshot).toHaveBeenCalledWith(3, expect.any(Function)); + }); + + it("chases a moving tip without waiting out the poll interval", async () => { + const { runner, pool, committer } = setup({ tipHeight: 1, pollIntervalMs: 60_000 }); + let tipCalls = 0; + pool.getTipHeight.mockImplementation(async () => { + tipCalls += 1; + return tipCalls === 1 ? 1 : 2; + }); + committer.commit.mockImplementation(async decoded => { + if (decoded.height >= 2) { + await runner.dispose(); + } + }); + + await runner.start(); + + expect(committer.commit).toHaveBeenCalledTimes(2); + }); + + it("keeps syncing when a staking snapshot fails", async () => { + const { runner, stakingSnapshot, committer, pool } = setup({ tipHeight: 1, pollIntervalMs: 1 }); + stakingSnapshot.snapshot.mockRejectedValueOnce(new Error("abci down")); + let tipCalls = 0; + pool.getTipHeight.mockImplementation(async () => { + tipCalls += 1; + return tipCalls === 1 ? 1 : 2; + }); + committer.commit.mockImplementation(async decoded => { + if (decoded.height >= 2) { + await runner.dispose(); + } + }); + + await runner.start(); + + expect(committer.commit).toHaveBeenCalledTimes(2); + expect(stakingSnapshot.snapshot).toHaveBeenCalled(); + }); + }); + function setup(input: { tipHeight: number; archiveEnabled?: boolean; @@ -142,13 +193,17 @@ describe(SyncRunnerService.name, () => { genesisImportEnabled?: boolean; checkpointHeight?: number; omitStartHeight?: boolean; + stopOn?: "commit" | "snapshot"; + pollIntervalMs?: number; }) { const archiveEnabled = input.archiveEnabled ?? true; + const stopOn = input.stopOn ?? "commit"; const config = envSchema.parse({ POSTGRES_DB_URI: "postgres://unit:unit@localhost:5432/unit", ...(input.omitStartHeight ? {} : { SYNC_START_HEIGHT: "1" }), ARCHIVE_BUCKET: archiveEnabled ? "raw-blocks" : "", - ...(input.genesisImportEnabled ? { GENESIS_IMPORT: "true" } : {}) + ...(input.genesisImportEnabled ? { GENESIS_IMPORT: "true" } : {}), + ...(input.pollIntervalMs !== undefined ? { SYNC_POLL_INTERVAL_MS: String(input.pollIntervalMs) } : {}) }); const dbFake = { @@ -185,15 +240,23 @@ describe(SyncRunnerService.name, () => { const committer = mock(); const genesisImport = mock(); + const stakingSnapshot = mock(); const logger = mock(); - const runner = new SyncRunnerService(dbFake as unknown as ChainDatabase, pool, decoder, committer, archive, genesisImport, config, logger); - committer.commit.mockImplementation(async decoded => { - if (decoded.height >= input.tipHeight) { + const runner = new SyncRunnerService(dbFake as unknown as ChainDatabase, pool, decoder, committer, archive, genesisImport, stakingSnapshot, config, logger); + if (stopOn === "commit") { + committer.commit.mockImplementation(async decoded => { + if (decoded.height >= input.tipHeight) { + await runner.dispose(); + } + }); + } + if (stopOn === "snapshot") { + stakingSnapshot.snapshot.mockImplementation(async () => { await runner.dispose(); - } - }); + }); + } - return { runner, archive, committer, genesisImport, logger, pool }; + return { runner, archive, committer, genesisImport, stakingSnapshot, logger, pool }; } function buildDecodedBlock(height: number): DecodedBlock { diff --git a/apps/chain-indexer/src/pipeline/sync-runner.service.ts b/apps/chain-indexer/src/pipeline/sync-runner.service.ts index e37f38afdc..be0616dc97 100644 --- a/apps/chain-indexer/src/pipeline/sync-runner.service.ts +++ b/apps/chain-indexer/src/pipeline/sync-runner.service.ts @@ -17,6 +17,7 @@ import type { ChainDatabase } from "@src/providers/db.provider"; import { CHAIN_DB } from "@src/providers/db.provider"; import { LoggerService } from "@src/providers/logging.provider"; import { RpcClientPool } from "@src/rpc/rpc-client-pool.service"; +import { StakingSnapshotService } from "@src/staking/staking-snapshot.service"; const PROGRESS_LOG_EVERY_BLOCKS = 100; @@ -28,11 +29,13 @@ export class SyncRunnerService { readonly #committer: BlockCommitterService; readonly #archive: BlockArchiveService; readonly #genesisImport: GenesisImportService; + readonly #stakingSnapshot: StakingSnapshotService; readonly #config: EnvConfig; readonly #logger: LoggerService; #stopped = false; #lastHash: Buffer | null = null; + #lastSnapshotHeight: number | null = null; constructor( @inject(CHAIN_DB) db: ChainDatabase, @@ -41,6 +44,7 @@ export class SyncRunnerService { @inject(BlockCommitterService) committer: BlockCommitterService, @inject(BlockArchiveService) archive: BlockArchiveService, @inject(GenesisImportService) genesisImport: GenesisImportService, + @inject(StakingSnapshotService) stakingSnapshot: StakingSnapshotService, @inject(APP_CONFIG) config: EnvConfig, @inject(LoggerService) logger: LoggerService ) { @@ -50,6 +54,7 @@ export class SyncRunnerService { this.#committer = committer; this.#archive = archive; this.#genesisImport = genesisImport; + this.#stakingSnapshot = stakingSnapshot; this.#config = config; this.#logger = logger; this.#logger.setContext("SYNC"); @@ -85,16 +90,24 @@ export class SyncRunnerService { while (!this.#stopped) { const tipHeight = await this.#retryTransient(() => this.#pool.getTipHeight(), { event: "SYNC_TIP_FETCH_RETRY" }); - if (nextHeight > tipHeight) { - await delay(this.#config.SYNC_POLL_INTERVAL_MS); + if (nextHeight <= tipHeight) { + while (nextHeight <= tipHeight && !this.#stopped) { + const height = nextHeight; + await this.#retryTransient(() => this.#syncBlock(height), { event: "SYNC_BLOCK_RETRY", height }); + nextHeight++; + } + if (this.#stopped) { + return; + } + await this.#maybeSnapshotStaking(nextHeight - 1); continue; } - while (nextHeight <= tipHeight && !this.#stopped) { - const height = nextHeight; - await this.#retryTransient(() => this.#syncBlock(height), { event: "SYNC_BLOCK_RETRY", height }); - nextHeight++; + await this.#maybeSnapshotStaking(nextHeight - 1); + if (this.#stopped) { + return; } + await delay(this.#config.SYNC_POLL_INTERVAL_MS); } } @@ -118,6 +131,28 @@ export class SyncRunnerService { return await retryTransient(operation, { isStopped: () => this.#stopped, logger: this.#logger, logContext }); } + /** + * Reconciles the validator set after the observed tip is committed, even if a newer block appears before + * the next poll — waiting until nextHeight > tip would skip the snapshot whenever live follow stays one + * block behind. Throttled to one run per configured block interval. A failed snapshot is logged and + * retried on the next interval rather than halting the sync it rides alongside. + */ + async #maybeSnapshotStaking(head: number): Promise { + if (!this.#config.STAKING_SNAPSHOT_ENABLED || head < 1) { + return; + } + if (this.#lastSnapshotHeight !== null && head - this.#lastSnapshotHeight < this.#config.STAKING_SNAPSHOT_INTERVAL_BLOCKS) { + return; + } + + try { + await this.#stakingSnapshot.snapshot(head, () => this.#stopped); + this.#lastSnapshotHeight = head; + } catch (error) { + this.#logger.error({ event: "STAKING_SNAPSHOT_FAILED", height: head, error }); + } + } + /** The raw payloads are archived before decode and commit, so no block is ever committed without being archived and raw blocks survive even decoder bugs. */ async #syncBlock(height: number): Promise { const record = await fetchRawBlock(this.#pool, height); diff --git a/apps/chain-indexer/src/rpc/rpc-client-pool.service.spec.ts b/apps/chain-indexer/src/rpc/rpc-client-pool.service.spec.ts index ec68268ff2..1ef7500202 100644 --- a/apps/chain-indexer/src/rpc/rpc-client-pool.service.spec.ts +++ b/apps/chain-indexer/src/rpc/rpc-client-pool.service.spec.ts @@ -71,6 +71,15 @@ describe(RpcClientPool.name, () => { expect(fetchMock).toHaveBeenCalledTimes(2); }); + it("uses a dispatcher so the connect timeout follows RPC_TIMEOUT_MS", async () => { + const { pool, fetchMock } = setup(); + fetchMock.mockResolvedValue(jsonResponse({ result: { sync_info: { latest_block_height: "1" } } })); + + await pool.getStatus(); + + expect(fetchMock.mock.calls[0][1]).toEqual(expect.objectContaining({ dispatcher: expect.any(Object) })); + }); + it("parses the tip height from the status payload", async () => { const { pool, fetchMock } = setup(); fetchMock.mockResolvedValue(jsonResponse({ result: { sync_info: { latest_block_height: "1234" } } })); diff --git a/apps/chain-indexer/src/rpc/rpc-client-pool.service.ts b/apps/chain-indexer/src/rpc/rpc-client-pool.service.ts index ee387a164d..622c847574 100644 --- a/apps/chain-indexer/src/rpc/rpc-client-pool.service.ts +++ b/apps/chain-indexer/src/rpc/rpc-client-pool.service.ts @@ -1,5 +1,6 @@ import { netConfig } from "@akashnetwork/net"; import { inject, singleton } from "tsyringe"; +import { Agent } from "undici"; import type { EnvConfig } from "@src/config/env.config"; import { APP_CONFIG } from "@src/providers/app-config.provider"; @@ -22,6 +23,11 @@ export class RpcClientPool { readonly #nodes: RpcNodeState[]; readonly #timeoutMs: number; readonly #cooldownMs: number; + /** + * AbortSignal.timeout only bounds the whole request. undici's default connect timeout is 10s and is + * independent of that signal, which is too short for a loaded archival node. + */ + readonly #dispatcher: Agent; readonly #logger: LoggerService; constructor(@inject(APP_CONFIG) config: EnvConfig, @inject(LoggerService) logger: LoggerService) { @@ -38,6 +44,7 @@ export class RpcClientPool { this.#nodes = endpoints.map(endpoint => ({ endpoint: endpoint.replace(/\/$/, ""), inFlight: 0, unhealthyUntil: 0 })); this.#timeoutMs = config.RPC_TIMEOUT_MS; this.#cooldownMs = config.RPC_NODE_COOLDOWN_MS; + this.#dispatcher = new Agent({ connectTimeout: this.#timeoutMs, headersTimeout: this.#timeoutMs, bodyTimeout: this.#timeoutMs }); this.#logger = logger; this.#logger.setContext("RPC_POOL"); } @@ -107,7 +114,7 @@ export class RpcClientPool { } async #fetchFromNode(endpoint: string, path: string): Promise { - const response = await fetch(`${endpoint}${path}`, { signal: AbortSignal.timeout(this.#timeoutMs) }); + const response = await fetch(`${endpoint}${path}`, { signal: AbortSignal.timeout(this.#timeoutMs), dispatcher: this.#dispatcher } as RequestInit); if (!response.ok) { throw new Error(`RPC responded with status ${response.status}`); diff --git a/apps/chain-indexer/src/staking/staking-query.spec.ts b/apps/chain-indexer/src/staking/staking-query.spec.ts new file mode 100644 index 0000000000..788b79d89a --- /dev/null +++ b/apps/chain-indexer/src/staking/staking-query.spec.ts @@ -0,0 +1,209 @@ +import { toBase64 } from "@cosmjs/encoding"; +import { QueryValidatorDelegationsResponse, QueryValidatorsResponse, QueryValidatorUnbondingDelegationsResponse } from "cosmjs-types/cosmos/staking/v1beta1/query"; +import { BondStatus } from "cosmjs-types/cosmos/staking/v1beta1/staking"; +import { describe, expect, it } from "vitest"; + +import { + decodeValidatorDelegations, + decodeValidators, + decodeValidatorUnbonding, + encodeValidatorDelegationsRequest, + encodeValidatorsRequest, + formatDec, + VALIDATOR_DELEGATIONS_PATH, + VALIDATORS_PATH +} from "@src/staking/staking-query"; + +describe("staking-query", () => { + describe("formatDec", () => { + it("renders a whole-share LegacyDec as a fixed 18-decimal string", () => { + expect(formatDec("5000000000000000000000000")).toBe("5000000.000000000000000000"); + }); + + it("renders a sub-one commission rate", () => { + expect(formatDec("100000000000000000")).toBe("0.100000000000000000"); + }); + + it("renders zero and negative values", () => { + expect(formatDec("0")).toBe("0.000000000000000000"); + expect(formatDec("-2500000000000000000")).toBe("-2.500000000000000000"); + }); + }); + + describe("encode", () => { + it("encodes a validators request as hex", () => { + expect(encodeValidatorsRequest()).toMatch(/^[0-9a-f]*$/); + expect(VALIDATORS_PATH).toBe("/cosmos.staking.v1beta1.Query/Validators"); + }); + + it("encodes a validator-delegations request carrying the operator address", () => { + const hex = encodeValidatorDelegationsRequest("akashvaloper1abc"); + + expect(Buffer.from(hex, "hex").toString("utf8")).toContain("akashvaloper1abc"); + expect(VALIDATOR_DELEGATIONS_PATH).toBe("/cosmos.staking.v1beta1.Query/ValidatorDelegations"); + }); + }); + + describe("decodeValidators", () => { + it("maps a bonded validator's dynamic state and scales its shares", () => { + const value = toBase64( + QueryValidatorsResponse.encode( + QueryValidatorsResponse.fromPartial({ + validators: [ + { + operatorAddress: "akashvaloper1abc", + jailed: false, + status: BondStatus.BOND_STATUS_BONDED, + tokens: "7000000", + delegatorShares: "7000000000000000000000000" + } + ] + }) + ).finish() + ); + + expect(decodeValidators(value).items).toEqual([ + { + operatorAddress: "akashvaloper1abc", + hexAddress: null, + accountAddress: null, + moniker: null, + identity: null, + website: null, + details: null, + securityContact: null, + commissionRate: null, + commissionMaxRate: null, + commissionMaxChangeRate: null, + minSelfDelegation: null, + jailed: false, + status: "bonded", + tokens: "7000000", + delegatorShares: "7000000.000000000000000000", + unbondingHeight: null, + unbondingTime: null + } + ]); + }); + + it("maps a validator's description and commission, scaling the rates", () => { + const value = toBase64( + QueryValidatorsResponse.encode( + QueryValidatorsResponse.fromPartial({ + validators: [ + { + operatorAddress: "akashvaloper1abc", + status: BondStatus.BOND_STATUS_BONDED, + tokens: "1", + delegatorShares: "1000000000000000000", + description: { moniker: "Node A", identity: "ABC", website: "https://a", details: "d", securityContact: "s@a" }, + commission: { commissionRates: { rate: "50000000000000000", maxRate: "200000000000000000", maxChangeRate: "10000000000000000" } }, + minSelfDelegation: "1" + } + ] + }) + ).finish() + ); + + expect(decodeValidators(value).items[0]).toMatchObject({ + moniker: "Node A", + identity: "ABC", + website: "https://a", + details: "d", + securityContact: "s@a", + commissionRate: "0.050000000000000000", + commissionMaxRate: "0.200000000000000000", + commissionMaxChangeRate: "0.010000000000000000", + minSelfDelegation: "1" + }); + }); + + it("keeps the unbonding height and time for an unbonding validator", () => { + const value = toBase64( + QueryValidatorsResponse.encode( + QueryValidatorsResponse.fromPartial({ + validators: [ + { + operatorAddress: "akashvaloper1unbonding", + jailed: true, + status: BondStatus.BOND_STATUS_UNBONDING, + tokens: "0", + delegatorShares: "0", + unbondingHeight: 4321n, + unbondingTime: { seconds: 1_700_000_000n, nanos: 0 } + } + ] + }) + ).finish() + ); + + const [validator] = decodeValidators(value).items; + + expect(validator.status).toBe("unbonding"); + expect(validator.jailed).toBe(true); + expect(validator.unbondingHeight).toBe(4321); + expect(validator.unbondingTime).toEqual(new Date(1_700_000_000_000)); + }); + + it("surfaces the pagination cursor when the page is full", () => { + const value = toBase64( + QueryValidatorsResponse.encode( + QueryValidatorsResponse.fromPartial({ validators: [], pagination: { nextKey: new Uint8Array([1, 2, 3]), total: 0n } }) + ).finish() + ); + + expect(decodeValidators(value).nextKey).toEqual(new Uint8Array([1, 2, 3])); + }); + + it("returns an empty page for an absent value", () => { + expect(decodeValidators(null)).toEqual({ items: [], nextKey: null }); + }); + }); + + describe("decodeValidatorDelegations", () => { + it("maps delegations under the queried validator and scales shares", () => { + const value = toBase64( + QueryValidatorDelegationsResponse.encode( + QueryValidatorDelegationsResponse.fromPartial({ + delegationResponses: [ + { delegation: { delegatorAddress: "akash1del", validatorAddress: "akashvaloper1abc", shares: "3000000000000000000000000" }, balance: { denom: "uakt", amount: "3000000" } } + ] + }) + ).finish() + ); + + expect(decodeValidatorDelegations("akashvaloper1abc", value).items).toEqual([ + { delegatorAddress: "akash1del", validatorOperatorAddress: "akashvaloper1abc", shares: "3000000.000000000000000000" } + ]); + }); + }); + + describe("decodeValidatorUnbonding", () => { + it("flattens unbonding entries under the delegator and queried validator", () => { + const value = toBase64( + QueryValidatorUnbondingDelegationsResponse.encode( + QueryValidatorUnbondingDelegationsResponse.fromPartial({ + unbondingResponses: [ + { + delegatorAddress: "akash1del", + validatorAddress: "akashvaloper1abc", + entries: [{ creationHeight: 100n, completionTime: { seconds: 1_700_000_500n, nanos: 0 }, initialBalance: "500000", balance: "500000" }] + } + ] + }) + ).finish() + ); + + expect(decodeValidatorUnbonding("akashvaloper1abc", value).items).toEqual([ + { + delegatorAddress: "akash1del", + validatorOperatorAddress: "akashvaloper1abc", + creationHeight: 100, + completionTime: new Date(1_700_000_500_000), + initialBalance: "500000", + balance: "500000" + } + ]); + }); + }); +}); diff --git a/apps/chain-indexer/src/staking/staking-query.ts b/apps/chain-indexer/src/staking/staking-query.ts new file mode 100644 index 0000000000..d97867eeb3 --- /dev/null +++ b/apps/chain-indexer/src/staking/staking-query.ts @@ -0,0 +1,218 @@ +import { fromBase64, toBase64, toHex } from "@cosmjs/encoding"; +import { PubKey as Ed25519PubKey } from "cosmjs-types/cosmos/crypto/ed25519/keys"; +import { + QueryValidatorDelegationsRequest, + QueryValidatorDelegationsResponse, + QueryValidatorsRequest, + QueryValidatorsResponse, + QueryValidatorUnbondingDelegationsRequest, + QueryValidatorUnbondingDelegationsResponse +} from "cosmjs-types/cosmos/staking/v1beta1/query"; +import { BondStatus } from "cosmjs-types/cosmos/staking/v1beta1/staking"; +import type { Any } from "cosmjs-types/google/protobuf/any"; +import type { Timestamp } from "cosmjs-types/google/protobuf/timestamp"; + +import { consensusHexAddress, operatorToAccountAddress } from "@src/genesis/genesis-address"; + +export const VALIDATORS_PATH = "/cosmos.staking.v1beta1.Query/Validators"; +export const VALIDATOR_DELEGATIONS_PATH = "/cosmos.staking.v1beta1.Query/ValidatorDelegations"; +export const VALIDATOR_UNBONDING_PATH = "/cosmos.staking.v1beta1.Query/ValidatorUnbondingDelegations"; + +/** One node page. Callers loop on the returned cursor, so this bounds a single round-trip, not the whole set. */ +const PAGE_LIMIT = 1000n; + +export type SnapshotValidatorStatus = "bonded" | "unbonding" | "unbonded"; + +/** The full validator row the snapshot reconciles against the chain, including the consensus `hexAddress` derived from the ed25519 pubkey. */ +export interface SnapshotValidator { + operatorAddress: string; + hexAddress: string | null; + accountAddress: string | null; + moniker: string | null; + identity: string | null; + website: string | null; + details: string | null; + securityContact: string | null; + commissionRate: string | null; + commissionMaxRate: string | null; + commissionMaxChangeRate: string | null; + minSelfDelegation: string | null; + jailed: boolean; + status: SnapshotValidatorStatus | null; + tokens: string; + delegatorShares: string; + unbondingHeight: number | null; + unbondingTime: Date | null; +} + +export interface SnapshotDelegation { + delegatorAddress: string; + validatorOperatorAddress: string; + shares: string; +} + +export interface SnapshotUnbondingEntry { + delegatorAddress: string; + validatorOperatorAddress: string; + creationHeight: number; + completionTime: Date; + initialBalance: string; + balance: string; +} + +/** A decoded page plus the cursor to fetch the next one, or null once the set is exhausted. */ +export interface Page { + items: T[]; + nextKey: Uint8Array | null; +} + +export function encodeValidatorsRequest(pageKey: Uint8Array = new Uint8Array()): string { + return toHex(QueryValidatorsRequest.encode(QueryValidatorsRequest.fromPartial({ status: "", pagination: pageRequest(pageKey) })).finish()); +} + +export function encodeValidatorDelegationsRequest(validatorAddr: string, pageKey: Uint8Array = new Uint8Array()): string { + return toHex(QueryValidatorDelegationsRequest.encode(QueryValidatorDelegationsRequest.fromPartial({ validatorAddr, pagination: pageRequest(pageKey) })).finish()); +} + +export function encodeValidatorUnbondingRequest(validatorAddr: string, pageKey: Uint8Array = new Uint8Array()): string { + return toHex(QueryValidatorUnbondingDelegationsRequest.encode(QueryValidatorUnbondingDelegationsRequest.fromPartial({ validatorAddr, pagination: pageRequest(pageKey) })).finish()); +} + +export function decodeValidators(value: string | null): Page { + if (!value) { + return { items: [], nextKey: null }; + } + const response = QueryValidatorsResponse.decode(fromBase64(value)); + return { + items: response.validators.map(validator => { + const rates = validator.commission?.commissionRates; + return { + operatorAddress: validator.operatorAddress, + hexAddress: toConsensusHexAddress(validator.consensusPubkey), + accountAddress: toAccountAddress(validator.operatorAddress), + moniker: emptyToNull(validator.description?.moniker), + identity: emptyToNull(validator.description?.identity), + website: emptyToNull(validator.description?.website), + details: emptyToNull(validator.description?.details), + securityContact: emptyToNull(validator.description?.securityContact), + commissionRate: decOrNull(rates?.rate), + commissionMaxRate: decOrNull(rates?.maxRate), + commissionMaxChangeRate: decOrNull(rates?.maxChangeRate), + minSelfDelegation: emptyToNull(validator.minSelfDelegation), + jailed: validator.jailed, + status: toStatus(validator.status), + tokens: validator.tokens || "0", + delegatorShares: formatDec(validator.delegatorShares), + unbondingHeight: validator.unbondingHeight > 0n ? Number(validator.unbondingHeight) : null, + unbondingTime: toDateOrNull(validator.unbondingTime) + }; + }), + nextKey: nextKeyOf(response.pagination?.nextKey) + }; +} + +export function decodeValidatorDelegations(validatorOperatorAddress: string, value: string | null): Page { + if (!value) { + return { items: [], nextKey: null }; + } + const response = QueryValidatorDelegationsResponse.decode(fromBase64(value)); + return { + items: response.delegationResponses.map(({ delegation }) => ({ + delegatorAddress: delegation.delegatorAddress, + validatorOperatorAddress, + shares: formatDec(delegation.shares) + })), + nextKey: nextKeyOf(response.pagination?.nextKey) + }; +} + +export function decodeValidatorUnbonding(validatorOperatorAddress: string, value: string | null): Page { + if (!value) { + return { items: [], nextKey: null }; + } + const response = QueryValidatorUnbondingDelegationsResponse.decode(fromBase64(value)); + return { + items: response.unbondingResponses.flatMap(unbonding => + unbonding.entries.map(entry => ({ + delegatorAddress: unbonding.delegatorAddress, + validatorOperatorAddress, + creationHeight: Number(entry.creationHeight), + completionTime: toDate(entry.completionTime), + initialBalance: entry.initialBalance, + balance: entry.balance + })) + ), + nextKey: nextKeyOf(response.pagination?.nextKey) + }; +} + +function pageRequest(key: Uint8Array) { + return { key, offset: 0n, limit: PAGE_LIMIT, countTotal: false, reverse: false }; +} + +function nextKeyOf(key: Uint8Array | undefined): Uint8Array | null { + return key && key.length > 0 ? key : null; +} + +function emptyToNull(value: string | undefined): string | null { + return value ? value : null; +} + +function decOrNull(raw: string | undefined): string | null { + return raw ? formatDec(raw) : null; +} + +/** Consensus hex address derived from the validator's ed25519 pubkey; a missing or malformed key yields null rather than aborting the snapshot. */ +function toConsensusHexAddress(consensusPubkey: Any | undefined): string | null { + if (!consensusPubkey) { + return null; + } + try { + return consensusHexAddress(consensusPubkey.typeUrl, toBase64(Ed25519PubKey.decode(consensusPubkey.value).key)); + } catch { + return null; + } +} + +/** Derives the `akash…` account address from the operator address; a malformed operator address yields null rather than aborting the snapshot. */ +function toAccountAddress(operatorAddress: string): string | null { + try { + return operatorToAccountAddress(operatorAddress); + } catch { + return null; + } +} + +function toStatus(status: BondStatus): SnapshotValidatorStatus | null { + switch (status) { + case BondStatus.BOND_STATUS_BONDED: + return "bonded"; + case BondStatus.BOND_STATUS_UNBONDING: + return "unbonding"; + case BondStatus.BOND_STATUS_UNBONDED: + return "unbonded"; + default: + return null; + } +} + +/** + * A cosmos `LegacyDec` is marshaled over protobuf as its value scaled by 10^18 with no decimal point + * (e.g. `"5000000000000000000000000"`). Render it as a fixed 18-decimal string so it matches the + * genesis-seeded `numeric(38,18)` shares rather than being stored 10^18 times too large. + */ +export function formatDec(raw: string): string { + const negative = raw.startsWith("-"); + const digits = (negative ? raw.slice(1) : raw).padStart(19, "0"); + const whole = digits.slice(0, -18); + const fraction = digits.slice(-18); + return `${negative ? "-" : ""}${whole}.${fraction}`; +} + +function toDate(timestamp: Timestamp): Date { + return new Date(Number(timestamp.seconds) * 1000 + Math.floor(timestamp.nanos / 1_000_000)); +} + +function toDateOrNull(timestamp: Timestamp | undefined): Date | null { + return timestamp && timestamp.seconds > 0n ? toDate(timestamp) : null; +} diff --git a/apps/chain-indexer/src/staking/staking-snapshot.service.spec.ts b/apps/chain-indexer/src/staking/staking-snapshot.service.spec.ts new file mode 100644 index 0000000000..64410efa80 --- /dev/null +++ b/apps/chain-indexer/src/staking/staking-snapshot.service.spec.ts @@ -0,0 +1,274 @@ +import { toBase64 } from "@cosmjs/encoding"; +import { PubKey as Ed25519PubKey } from "cosmjs-types/cosmos/crypto/ed25519/keys"; +import { + QueryValidatorDelegationsResponse, + QueryValidatorsResponse, + QueryValidatorUnbondingDelegationsResponse +} from "cosmjs-types/cosmos/staking/v1beta1/query"; +import { BondStatus } from "cosmjs-types/cosmos/staking/v1beta1/staking"; +import { describe, expect, it } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import { Delegations, UnbondingDelegations, Validators } from "@src/db/schema"; +import type { AccountInterner } from "@src/pipeline/balance/account-interner.service"; +import type { ChainDatabase } from "@src/providers/db.provider"; +import type { LoggerService } from "@src/providers/logging.provider"; +import type { RpcClientPool } from "@src/rpc/rpc-client-pool.service"; +import { VALIDATOR_DELEGATIONS_PATH, VALIDATOR_UNBONDING_PATH, VALIDATORS_PATH } from "@src/staking/staking-query"; +import { StakingSnapshotService } from "@src/staking/staking-snapshot.service"; + +import { rowsFor } from "@test/fakes/build-tx-fake"; + +const OPERATOR = "akashvaloper1abc"; + +describe(StakingSnapshotService.name, () => { + it("upserts each validator's bond state at the snapshot height", async () => { + const { service, inserts } = setup({ + [VALIDATORS_PATH]: [ + validatorsResponse([ + validator({ operatorAddress: OPERATOR, status: BondStatus.BOND_STATUS_BONDED, tokens: "7000000", delegatorShares: "7000000000000000000000000" }) + ]) + ] + }); + + await service.snapshot(1000); + + expect(rowsFor(inserts, Validators)).toEqual([ + expect.objectContaining({ + operatorAddress: OPERATOR, + jailed: false, + status: "bonded", + tokens: "7000000", + delegatorShares: "7000000.000000000000000000", + unbondingHeight: null, + unbondingTime: null + }) + ]); + }); + + it("derives each validator's consensus hex address from its ed25519 pubkey", async () => { + const { service, inserts } = setup({ + [VALIDATORS_PATH]: [ + validatorsResponse([ + validator({ + operatorAddress: OPERATOR, + consensusPubkey: { typeUrl: "/cosmos.crypto.ed25519.PubKey", value: Ed25519PubKey.encode({ key: new Uint8Array(32).fill(1) }).finish() } + }) + ]) + ] + }); + + await service.snapshot(1000); + + expect(rowsFor(inserts, Validators)[0]).toMatchObject({ operatorAddress: OPERATOR, hexAddress: "72CD6E8422C407FB6D098690F1130B7DED7EC2F7" }); + }); + + it("leaves the consensus hex address null when the validator has no pubkey", async () => { + const { service, inserts } = setup({ [VALIDATORS_PATH]: [validatorsResponse([validator({ operatorAddress: OPERATOR })])] }); + + await service.snapshot(1000); + + expect(rowsFor(inserts, Validators)[0]).toMatchObject({ operatorAddress: OPERATOR, hexAddress: null }); + }); + + it("fully replaces delegations with interned delegator ids", async () => { + const { service, inserts, deletes } = setup({ + [VALIDATORS_PATH]: [validatorsResponse([validator({ operatorAddress: OPERATOR })])], + [VALIDATOR_DELEGATIONS_PATH]: [delegationsResponse([{ delegatorAddress: "akash1del", validatorAddress: OPERATOR, shares: "3000000000000000000000000" }])] + }); + + await service.snapshot(1000); + + expect(deletes).toContain(Delegations); + expect(rowsFor(inserts, Delegations)).toEqual([{ delegatorAccountId: 1, validatorOperatorAddress: OPERATOR, shares: "3000000.000000000000000000" }]); + }); + + it("fully replaces unbonding entries with interned delegator ids", async () => { + const { service, inserts, deletes } = setup({ + [VALIDATORS_PATH]: [validatorsResponse([validator({ operatorAddress: OPERATOR })])], + [VALIDATOR_UNBONDING_PATH]: [ + unbondingResponse([ + { + delegatorAddress: "akash1del", + validatorAddress: OPERATOR, + entries: [{ creationHeight: 900n, completionTime: { seconds: 1_700_000_000n, nanos: 0 }, initialBalance: "500000", balance: "500000" }] + } + ]) + ] + }); + + await service.snapshot(1000); + + expect(deletes).toContain(UnbondingDelegations); + expect(rowsFor(inserts, UnbondingDelegations)).toEqual([ + { + delegatorAccountId: 1, + validatorOperatorAddress: OPERATOR, + creationHeight: 900, + completionTime: new Date(1_700_000_000_000), + initialBalance: "500000", + balance: "500000" + } + ]); + }); + + it("follows the pagination cursor across validator pages", async () => { + const { service, inserts } = setup({ + [VALIDATORS_PATH]: [ + validatorsResponse([validator({ operatorAddress: "akashvaloper1a" })], new Uint8Array([9])), + validatorsResponse([validator({ operatorAddress: "akashvaloper1b" })]) + ] + }); + + await service.snapshot(1000); + + expect(rowsFor(inserts, Validators).map(row => row.operatorAddress)).toEqual(["akashvaloper1a", "akashvaloper1b"]); + }); + + it("fetches and writes delegations for every validator in the set", async () => { + const { service, inserts } = setup({ + [VALIDATORS_PATH]: [validatorsResponse([validator({ operatorAddress: "akashvaloper1a" }), validator({ operatorAddress: "akashvaloper1b" })])], + [VALIDATOR_DELEGATIONS_PATH]: [ + delegationsResponse([{ delegatorAddress: "akash1dela", validatorAddress: "akashvaloper1a", shares: "1000000000000000000" }]), + delegationsResponse([{ delegatorAddress: "akash1delb", validatorAddress: "akashvaloper1b", shares: "2000000000000000000" }]) + ] + }); + + await service.snapshot(1000); + + expect(rowsFor(inserts, Delegations)).toEqual([ + { delegatorAccountId: 1, validatorOperatorAddress: "akashvaloper1a", shares: "1.000000000000000000" }, + { delegatorAccountId: 2, validatorOperatorAddress: "akashvaloper1b", shares: "2.000000000000000000" } + ]); + }); + + it("interns every delegator before writing", async () => { + const { service, interner } = setup({ + [VALIDATORS_PATH]: [validatorsResponse([validator({ operatorAddress: OPERATOR })])], + [VALIDATOR_DELEGATIONS_PATH]: [delegationsResponse([{ delegatorAddress: "akash1del", validatorAddress: OPERATOR, shares: "1000000000000000000" }])], + [VALIDATOR_UNBONDING_PATH]: [ + unbondingResponse([ + { + delegatorAddress: "akash1unbonding", + validatorAddress: OPERATOR, + entries: [{ creationHeight: 1n, completionTime: { seconds: 1n, nanos: 0 }, initialBalance: "1", balance: "1" }] + } + ]) + ] + }); + + await service.snapshot(1000); + + expect(new Set([...interner.resolve.mock.calls[0][0]])).toEqual(new Set(["akash1del", "akash1unbonding"])); + }); + + it("aborts the fetch when told to stop", async () => { + const { service } = setup({ + [VALIDATORS_PATH]: [validatorsResponse([validator({ operatorAddress: OPERATOR })])] + }); + + await expect(service.snapshot(1000, () => true)).rejects.toThrow("Staking snapshot stopped"); + }); + + it("retries a transient staking query failure and still writes the page", async () => { + const { service, inserts, rpc } = setup({ + [VALIDATORS_PATH]: [validatorsResponse([validator({ operatorAddress: OPERATOR })])] + }); + rpc.abciQuery.mockRejectedValueOnce(new Error("ECONNRESET")); + + await service.snapshot(1000); + + expect(rowsFor(inserts, Validators)).toEqual([expect.objectContaining({ operatorAddress: OPERATOR })]); + }); + + it("skips all writes when the chain reports no validators", async () => { + const { service, inserts, deletes } = setup({ [VALIDATORS_PATH]: [validatorsResponse([])] }); + + await service.snapshot(1000); + + expect(inserts).toEqual([]); + expect(deletes).toEqual([]); + }); + + function setup(responsesByPath: Record) { + const inserts: { table: unknown; rows: Record[] }[] = []; + const deletes: unknown[] = []; + + const txFake = { + insert: (table: unknown) => ({ + values: (rows: Record | Record[]) => { + inserts.push({ table, rows: Array.isArray(rows) ? rows : [rows] }); + return Object.assign(Promise.resolve(), { + onConflictDoNothing: () => Object.assign(Promise.resolve(), { returning: () => Promise.resolve([]) }), + onConflictDoUpdate: () => Promise.resolve() + }); + } + }), + delete: (table: unknown) => { + deletes.push(table); + return Promise.resolve(); + } + }; + const dbFake = { transaction: (callback: (tx: unknown) => Promise) => callback(txFake) }; + + const callsByPath: Record = {}; + const rpc = mock(); + rpc.abciQuery.mockImplementation(async path => { + const queue = responsesByPath[path] ?? []; + const index = callsByPath[path] ?? 0; + callsByPath[path] = index + 1; + return { value: queue[index] ?? null }; + }); + + const interner = mock(); + interner.resolve.mockImplementation(async addresses => new Map([...addresses].map((address, index) => [address, index + 1]))); + + const service = new StakingSnapshotService(dbFake as unknown as ChainDatabase, rpc, interner, mock()); + return { service, inserts, deletes, rpc, interner }; + } +}); + +function validator(overrides: { + operatorAddress: string; + status?: BondStatus; + tokens?: string; + delegatorShares?: string; + consensusPubkey?: { typeUrl: string; value: Uint8Array }; +}) { + return { + operatorAddress: overrides.operatorAddress, + consensusPubkey: overrides.consensusPubkey, + jailed: false, + status: overrides.status ?? BondStatus.BOND_STATUS_BONDED, + tokens: overrides.tokens ?? "0", + delegatorShares: overrides.delegatorShares ?? "0" + }; +} + +function validatorsResponse(validators: ReturnType[], nextKey?: Uint8Array): string { + return toBase64( + QueryValidatorsResponse.encode(QueryValidatorsResponse.fromPartial({ validators, pagination: nextKey ? { nextKey, total: 0n } : undefined })).finish() + ); +} + +function delegationsResponse(delegations: { delegatorAddress: string; validatorAddress: string; shares: string }[]): string { + return toBase64( + QueryValidatorDelegationsResponse.encode( + QueryValidatorDelegationsResponse.fromPartial({ + delegationResponses: delegations.map(delegation => ({ delegation, balance: { denom: "uakt", amount: "0" } })) + }) + ).finish() + ); +} + +function unbondingResponse( + unbonding: { + delegatorAddress: string; + validatorAddress: string; + entries: { creationHeight: bigint; completionTime: { seconds: bigint; nanos: number }; initialBalance: string; balance: string }[]; + }[] +): string { + return toBase64( + QueryValidatorUnbondingDelegationsResponse.encode(QueryValidatorUnbondingDelegationsResponse.fromPartial({ unbondingResponses: unbonding })).finish() + ); +} diff --git a/apps/chain-indexer/src/staking/staking-snapshot.service.ts b/apps/chain-indexer/src/staking/staking-snapshot.service.ts new file mode 100644 index 0000000000..edf735c866 --- /dev/null +++ b/apps/chain-indexer/src/staking/staking-snapshot.service.ts @@ -0,0 +1,244 @@ +import { sql } from "drizzle-orm"; +import chunk from "lodash/chunk"; +import { inject, singleton } from "tsyringe"; + +import { INSERT_CHUNK_SIZE } from "@src/db/insert-chunk-size"; +import { insertChunked } from "@src/db/insert-chunked"; +import { Delegations, UnbondingDelegations, Validators } from "@src/db/schema"; +import { retryWithBackoff } from "@src/lib/retry-with-backoff/retry-with-backoff"; +import { AccountInterner } from "@src/pipeline/balance/account-interner.service"; +import type { ChainDatabase, ChainTransaction } from "@src/providers/db.provider"; +import { CHAIN_DB } from "@src/providers/db.provider"; +import { LoggerService } from "@src/providers/logging.provider"; +import { RpcClientPool } from "@src/rpc/rpc-client-pool.service"; +import type { Page, SnapshotDelegation, SnapshotUnbondingEntry, SnapshotValidator } from "@src/staking/staking-query"; +import { + decodeValidatorDelegations, + decodeValidators, + decodeValidatorUnbonding, + encodeValidatorDelegationsRequest, + encodeValidatorsRequest, + encodeValidatorUnbondingRequest, + VALIDATOR_DELEGATIONS_PATH, + VALIDATOR_UNBONDING_PATH, + VALIDATORS_PATH +} from "@src/staking/staking-query"; + +/** Validators whose delegations and unbonding are fetched at once. Kept small so a single archival node is not flooded with parallel TLS handshakes. */ +const SNAPSHOT_FETCH_CONCURRENCY = 2; +const SNAPSHOT_QUERY_RETRIES = 5; +const SNAPSHOT_QUERY_RETRY_BASE_MS = 1_000; + +/** + * Reconciles validators, delegations and unbonding to the chain's own answer, because delegation *shares* + * cannot be derived exactly from messages — the token↔share rate drifts with every reward and slash. The + * whole set is fetched before any write, so a failed query aborts the run without touching the tables. + * Delegations and unbonding are fully replaced (an entry vanishes silently once it matures or fully + * undelegates); validators are upserted in full from the chain query, including descriptor, commission + * and addresses. + */ +@singleton() +export class StakingSnapshotService { + readonly #db: ChainDatabase; + readonly #rpc: RpcClientPool; + readonly #interner: AccountInterner; + readonly #logger: LoggerService; + + constructor( + @inject(CHAIN_DB) db: ChainDatabase, + @inject(RpcClientPool) rpc: RpcClientPool, + @inject(AccountInterner) interner: AccountInterner, + @inject(LoggerService) logger: LoggerService + ) { + this.#db = db; + this.#rpc = rpc; + this.#interner = interner; + this.#logger = logger; + this.#logger.setContext("STAKING_SNAPSHOT"); + } + + async snapshot(height: number, isStopped: () => boolean = () => false): Promise { + const validators = await this.#fetchAll(height, VALIDATORS_PATH, encodeValidatorsRequest, decodeValidators, isStopped); + if (validators.length === 0) { + this.#logger.warn({ event: "STAKING_SNAPSHOT_NO_VALIDATORS", height }); + return; + } + + const delegations: SnapshotDelegation[] = []; + const unbonding: SnapshotUnbondingEntry[] = []; + for (const batch of chunk(validators, SNAPSHOT_FETCH_CONCURRENCY)) { + const stakes = await Promise.all(batch.map(({ operatorAddress }) => this.#fetchValidatorStake(height, operatorAddress, isStopped))); + for (const stake of stakes) { + delegations.push(...stake.delegations); + unbonding.push(...stake.unbonding); + } + } + + const accountIds = await this.#interner.resolve([ + ...delegations.map(delegation => delegation.delegatorAddress), + ...unbonding.map(entry => entry.delegatorAddress) + ]); + + await this.#db.transaction(async tx => { + await this.#upsertValidators(tx, validators); + await this.#replaceDelegations(tx, delegations, accountIds); + await this.#replaceUnbonding(tx, unbonding, accountIds); + }); + + this.#logger.info({ + event: "STAKING_SNAPSHOT_WRITTEN", + height, + validators: validators.length, + delegations: delegations.length, + unbonding: unbonding.length + }); + } + + /** A validator's delegations and unbonding entries fetched concurrently; both are read-only and pinned to `height`. */ + async #fetchValidatorStake( + height: number, + operatorAddress: string, + isStopped: () => boolean + ): Promise<{ delegations: SnapshotDelegation[]; unbonding: SnapshotUnbondingEntry[] }> { + const [delegations, unbonding] = await Promise.all([ + this.#fetchAll( + height, + VALIDATOR_DELEGATIONS_PATH, + key => encodeValidatorDelegationsRequest(operatorAddress, key), + value => decodeValidatorDelegations(operatorAddress, value), + isStopped + ), + this.#fetchAll( + height, + VALIDATOR_UNBONDING_PATH, + key => encodeValidatorUnbondingRequest(operatorAddress, key), + value => decodeValidatorUnbonding(operatorAddress, value), + isStopped + ) + ]); + return { delegations, unbonding }; + } + + /** Walks a paginated staking query to exhaustion, following the node's `next_key` cursor across pages. */ + async #fetchAll( + height: number, + path: string, + encode: (key: Uint8Array) => string, + decode: (value: string | null) => Page, + isStopped: () => boolean + ): Promise { + const items: T[] = []; + let key: Uint8Array = new Uint8Array(); + + for (;;) { + if (isStopped()) { + throw new Error("Staking snapshot stopped"); + } + const response = await retryWithBackoff(() => this.#rpc.abciQuery(path, encode(key), height), { + maxAttempts: SNAPSHOT_QUERY_RETRIES, + baseDelayMs: SNAPSHOT_QUERY_RETRY_BASE_MS, + shouldRethrow: () => isStopped(), + onRetry: (error, attempt, delayMs) => this.#logger.warn({ event: "STAKING_SNAPSHOT_QUERY_RETRY", path, height, attempt, delayMs, error }) + }); + const page = decode(response.value); + items.push(...page.items); + if (!page.nextKey) { + return items; + } + key = page.nextKey; + } + } + + /** Upserts the full validator row, including the consensus `hexAddress` derived from the query's pubkey, so post-genesis validators are not left with a null one. */ + async #upsertValidators(tx: ChainTransaction, validators: SnapshotValidator[]): Promise { + const rows = validators.map(validator => ({ + operatorAddress: validator.operatorAddress, + hexAddress: validator.hexAddress, + accountAddress: validator.accountAddress, + moniker: validator.moniker, + identity: validator.identity, + website: validator.website, + details: validator.details, + securityContact: validator.securityContact, + commissionRate: validator.commissionRate, + commissionMaxRate: validator.commissionMaxRate, + commissionMaxChangeRate: validator.commissionMaxChangeRate, + minSelfDelegation: validator.minSelfDelegation, + jailed: validator.jailed, + status: validator.status, + tokens: validator.tokens, + delegatorShares: validator.delegatorShares, + unbondingHeight: validator.unbondingHeight, + unbondingTime: validator.unbondingTime + })); + + for (const rowChunk of chunk(rows, INSERT_CHUNK_SIZE)) { + await tx + .insert(Validators) + .values(rowChunk) + .onConflictDoUpdate({ + target: Validators.operatorAddress, + set: { + hexAddress: sql`excluded.hex_address`, + accountAddress: sql`excluded.account_address`, + moniker: sql`excluded.moniker`, + identity: sql`excluded.identity`, + website: sql`excluded.website`, + details: sql`excluded.details`, + securityContact: sql`excluded.security_contact`, + commissionRate: sql`excluded.commission_rate`, + commissionMaxRate: sql`excluded.commission_max_rate`, + commissionMaxChangeRate: sql`excluded.commission_max_change_rate`, + minSelfDelegation: sql`excluded.min_self_delegation`, + jailed: sql`excluded.jailed`, + status: sql`excluded.status`, + tokens: sql`excluded.tokens`, + delegatorShares: sql`excluded.delegator_shares`, + unbondingHeight: sql`excluded.unbonding_height`, + unbondingTime: sql`excluded.unbonding_time` + } + }); + } + } + + /** + * Delete-then-insert, ignoring PK collisions so two overlapping snapshots (rolling deploy) duplicate + * work instead of aborting. A DELETE that started before the other writer committed will not see those + * new rows; the insert then no-ops on the same keys rather than raising 23505. + */ + async #replaceDelegations(tx: ChainTransaction, delegations: SnapshotDelegation[], accountIds: Map): Promise { + await tx.delete(Delegations); + + const rows = delegations.map(delegation => ({ + delegatorAccountId: this.#requireId(accountIds, delegation.delegatorAddress), + validatorOperatorAddress: delegation.validatorOperatorAddress, + shares: delegation.shares + })); + + await insertChunked(tx, Delegations, rows); + } + + /** Same delete-then-insert as delegations: vanished entries disappear, concurrent writers do not abort. */ + async #replaceUnbonding(tx: ChainTransaction, unbonding: SnapshotUnbondingEntry[], accountIds: Map): Promise { + await tx.delete(UnbondingDelegations); + + const rows = unbonding.map(entry => ({ + delegatorAccountId: this.#requireId(accountIds, entry.delegatorAddress), + validatorOperatorAddress: entry.validatorOperatorAddress, + creationHeight: entry.creationHeight, + completionTime: entry.completionTime, + initialBalance: entry.initialBalance, + balance: entry.balance + })); + + await insertChunked(tx, UnbondingDelegations, rows); + } + + #requireId(accountIds: Map, address: string): number { + const accountId = accountIds.get(address); + if (accountId === undefined) { + throw new Error(`No interned account id for delegator ${address}`); + } + return accountId; + } +} diff --git a/package-lock.json b/package-lock.json index 97a6c32042..26f51b6bd1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -557,6 +557,7 @@ "postgres": "^3.4.4", "reflect-metadata": "^0.2.2", "tsyringe": "^4.10.0", + "undici": "^7.22.0", "zod": "3.*" }, "devDependencies": { From 399ca3a2e71b276573fd0f952f93b042f61a192f Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Sat, 15 Aug 2026 22:01:28 +0530 Subject: [PATCH 08/14] feat(indexer): akash proto type catalog with exhaustiveness guard and dead-letter store (#3598) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(indexer): snapshot cosmos validators, delegations and unbonding Reconcile the validator set against the chain's own staking queries so it matches at the tip. Delegation shares can't be derived exactly from messages, since the token/share exchange rate moves with every reward and slash, so an authoritative snapshot is the only way to keep them exact. - validators gains bond state (status, jailed, tokens, delegator_shares, unbonding height/time); new unbonding_delegations table; validator_status enum - StakingSnapshotService fetches the full set over abci_query, then upserts each validator's row and fully replaces delegations and unbonding in one transaction, interning delegators first - sync triggers a snapshot only once caught up to the tip, throttled by STAKING_SNAPSHOT_INTERVAL_BLOCKS, and never lets a snapshot failure halt sync Part of CON-810. * feat(indexer): index cosmos governance and sharpen distribution reasons Index governance proposals, votes and deposits per block, and tighten the balance-ledger reason for distribution flows. - proposals/proposal_votes/proposal_deposits tables plus proposal_status and vote_option enums - gov deriver reads MsgSubmitProposal (v1 and v1beta1), MsgVote/MsgVoteWeighted and MsgDeposit, taking each proposal's id from its submit_proposal event and its lifecycle from the EndBlock active_proposal/inactive_proposal events; a vote promotes its proposal into voting_period without regressing a result - GovWriter persists them inside the block transaction, resolving proposer, voter and depositor from the account ids the committer already interned (governance actors are always the message signer) - decoder now keeps the gov events; committer calls the gov writer - classifyReason is direction-aware: a credit from the distribution module is a reward/commission, but a fund-community-pool debit is a plain transfer, not a reward Part of CON-810. * fix(indexer): keep same-block gov deposits and track running total deposit The proposal_deposits primary key (proposal, depositor, height) dropped a second same-block deposit from the same depositor under onConflictDoNothing, and proposals.total_deposit was frozen at the submission-time initial deposit. Sum a depositor's same-block deposits in the deriver so none are lost, and recompute total_deposit from the full deposit history after each write so the running total stays correct across blocks and re-committing a block never double-counts. * fix(indexer): dedupe governance votes before the batch upsert A voter can re-vote on the same proposal within one commit batch, either via two MsgVote in a single block or across blocks in a backfill batch, which cosmos allows during the voting period. Both rows share the (proposal_id, voter_account_id) conflict target, so a single ON CONFLICT DO UPDATE over them fails with Postgres 21000 and aborts the whole block-committer transaction. Collapse votes to the latest by height before the upsert, the way aggregateDeposits already does for deposits. * refactor(indexer): fetch validator stake in bounded concurrent batches The staking snapshot fetched each validator's delegations and unbonding one query at a time, so a large validator set serialized hundreds of paginated abci_query round trips and held up the poll loop's next-block pickup for the snapshot's whole duration. Fetch a validator's delegations and unbonding concurrently and process validators in bounded batches. The queries are read-only and pinned to the snapshot height, and the RPC pool already balances concurrent load across nodes. * fix(indexer): skip governance messages from failed transactions deriveGovChanges processed every decoded message regardless of its transaction's result code. A MsgVote or MsgDeposit in a reverted tx (code != 0) still produced a vote upsert, a voting_period promotion, and a deposit row that permanently inflated Proposals.total_deposit, even though cosmos rolled the whole tx back and emitted no handler events for it. The vote and deposit paths read the message body directly rather than correlating against an event, so they had no signal the tx failed. Skip a transaction's messages when its code is non-zero, matching the event-driven balance deriver that is already correct for failed txs. * fix(indexer): guard the vote upsert against out-of-order commits writeVotes upserted last-writer-wins on (proposalId, voterAccountId) with no height check, so if two overlapping writers (e.g. pods during a rolling deploy, which block-committer documents as tolerated) commit a voter's re-votes out of height order, the older-height commit silently overwrote the newer vote. dedupeVotes only collapses re-votes merged into a single batch. Add a setWhere requiring excluded.height >= proposal_votes.height, mirroring the GREATEST() guard the committer already uses for the checkpoint, so a stale vote can no longer regress a newer one. * fix(indexer): derive validator consensus hex address in the staking snapshot upsertValidators left hex_address to genesis, so a validator created after genesis — whose row only ever comes from the periodic snapshot — kept a null consensus address forever. Derive it from the query's ed25519 consensus pubkey (the same value genesis computes) and include it in the upsert, so every validator the snapshot sees gets its hex_address. * refactor(indexer): sum proposal deposits in linear time refreshTotalDeposits rebuilt each proposal's amount array by spreading the whole accumulated array on every deposit row, which is O(n^2) in the deposit count. Push into the existing array and sum once instead: same result, linear time. * feat(indexer): akash proto type catalog with exhaustiveness guard and dead-letter store Message decoding previously collapsed unregistered types, corrupt bytes and oversized bodies into a silent null body, the same failure mode that makes chain upgrades an emergency in the old indexer. Decoding now runs off a declarative type catalog covering every Akash module of the installed chain SDK plus the historical versions from the frozen akash-api package, a unit test fails when an SDK bump ships an Akash type that is neither registered nor ignored, and messages that still fail to decode land in cosmos.message_dead_letters with their raw bytes and error while the block commits and sync keeps going. Re-running a backfill with BACKFILL_REPLAY=true after registering the type fills the null bodies and clears the healed dead letters. Closes CON-806 * fix(indexer): skip dead letters when the message body is already set The messages upsert is heal-only: a writer that fails to decode cannot overwrite a body another writer already filled in. The dead-letter write had no matching guard. After a range delete it inserted this writer's decode failures even when the body was already populated, so a draining pod with a stale type catalog could put a phantom dead-letter row back and fire MESSAGES_DEAD_LETTERED. Only insert a dead letter when messages.body is still null after the upsert, and only log the rows that actually landed. The README now matches type-catalog.ts and says the legacy catalog goes through v1beta4. --- apps/chain-indexer/README.md | 8 + .../drizzle/0005_mysterious_medusa.sql | 12 + .../drizzle/meta/0005_snapshot.json | 1283 +++++++++++++++++ apps/chain-indexer/drizzle/meta/_journal.json | 7 + apps/chain-indexer/env/.env.sample | 1 + apps/chain-indexer/package.json | 1 + .../src/config/env.config.spec.ts | 16 + apps/chain-indexer/src/config/env.config.ts | 6 + apps/chain-indexer/src/db/schema.spec.ts | 28 +- apps/chain-indexer/src/db/schema.ts | 22 + apps/chain-indexer/src/db/sql-excluded.ts | 7 + .../src/gov/gov-writer.service.ts | 9 +- .../src/http-schemas/status.schema.ts | 11 +- .../src/pipeline/backfill-planner.spec.ts | 20 + .../src/pipeline/backfill-planner.ts | 7 +- .../src/pipeline/backfill-runner.service.ts | 7 +- .../pipeline/block-committer.service.spec.ts | 123 +- .../src/pipeline/block-committer.service.ts | 125 +- .../pipeline/block-decoder.service.spec.ts | 44 +- .../src/pipeline/block-decoder.service.ts | 23 +- .../src/pipeline/decoded-block.ts | 7 + .../src/proto/akash-type-coverage.spec.ts | 67 + .../src/proto/type-catalog.spec.ts | 40 + apps/chain-indexer/src/proto/type-catalog.ts | 86 ++ .../src/providers/type-registry.provider.ts | 27 +- .../services/status/status.service.spec.ts | 59 + .../src/services/status/status.service.ts | 19 +- package-lock.json | 1 + 28 files changed, 1991 insertions(+), 75 deletions(-) create mode 100644 apps/chain-indexer/drizzle/0005_mysterious_medusa.sql create mode 100644 apps/chain-indexer/drizzle/meta/0005_snapshot.json create mode 100644 apps/chain-indexer/src/db/sql-excluded.ts create mode 100644 apps/chain-indexer/src/proto/akash-type-coverage.spec.ts create mode 100644 apps/chain-indexer/src/proto/type-catalog.spec.ts create mode 100644 apps/chain-indexer/src/proto/type-catalog.ts create mode 100644 apps/chain-indexer/src/services/status/status.service.spec.ts diff --git a/apps/chain-indexer/README.md b/apps/chain-indexer/README.md index 8f41264b2d..57e24f22c5 100644 --- a/apps/chain-indexer/README.md +++ b/apps/chain-indexer/README.md @@ -58,6 +58,14 @@ BACKFILL_TO_HEIGHT=200000 Blocks are fetched from RPC in parallel (`BACKFILL_CONCURRENCY`, default 10) and committed strictly in order in batches of `BACKFILL_BATCH_SIZE` blocks (default 200), each batch in one Postgres transaction together with the checkpoint advance. Progress is checkpointed per range under the `indexer_state` stream `backfill:{from}-{to}`, so killing and restarting the job resumes at the checkpoint without gaps or duplicates, and re-running a completed range exits 0 immediately. Changing the range creates a fresh checkpoint row. All inserts are natural-keyed and conflict-ignoring, so a backfill can run against the same database as live sync, and a duplicate backfill pod on the same range is harmless. +## Proto type catalog and dead letters + +Every message the decoder sees falls into one of three buckets, decided by `src/proto/type-catalog.ts`. Registered types decode to canonical JSON in `messages.body`; the catalog covers all Akash modules of the installed chain SDK plus the historical `v1beta1` through `v1beta4` versions from the frozen `@akashnetwork/akash-api` package, so mainnet history decodes too. Ignored types (each with a documented reason, e.g. cosmwasm on sandbox) store a null body and nothing else. Anything else is dead-lettered: the row in `messages` keeps its null body, and `message_dead_letters` records the raw bytes and the error, in the same transaction as the block, so ingestion never stalls on an unknown type. Each batch that dead-letters something logs a single `MESSAGES_DEAD_LETTERED` error with per-type counts, and `GET /v1/status` reports the store's totals; an alert can watch either signal. + +A unit test (`src/proto/akash-type-coverage.spec.ts`) enumerates every Akash type in the installed chain SDK and fails when one is neither registered nor ignored, which keeps a dependency bump that ships new types from merging unhandled. When it fires, either add the module to the catalog or put the new types on the ignore list with a reason. + +Dead letters heal by replay. Register the type (usually by bumping the SDK and updating the catalog), then re-run the backfill range with `BACKFILL_REPLAY=true`: the planner ignores the range's completed checkpoint, messages whose body was null get the decoded body on conflict, and each re-committed height clears its dead-letter rows. Rows that already had a body are left untouched, so a replay is cheap and idempotent. A writer that still fails to decode will not insert a dead letter for a message whose body is already set. + ## Balance ledger and activity log Every committed block also derives a balance ledger and an address activity log, in the same transaction as the block, so they never drift from the chain data they come from. `balance_changes` is the append-only ledger: one row per coin movement with the running `balance_after` and a classified `reason` (`mint`, `burn`, `slash`, `fee`, `reward`, `commission`, `staking`, `gov`, `ibc`, `escrow`, `bme`, or a plain `transfer`; genesis seeds are `genesis`). `account_balances` holds the current per-account per-denom balance, upserted from the ledger. `account_txs` is the activity log linking each account to the transactions that touched it. Addresses are interned to ids on first sight (`accounts`), so both live sync and backfill produce identical ledger rows for the same height. diff --git a/apps/chain-indexer/drizzle/0005_mysterious_medusa.sql b/apps/chain-indexer/drizzle/0005_mysterious_medusa.sql new file mode 100644 index 0000000000..c7c0a0f473 --- /dev/null +++ b/apps/chain-indexer/drizzle/0005_mysterious_medusa.sql @@ -0,0 +1,12 @@ +CREATE TABLE "cosmos"."message_dead_letters" ( + "height" bigint NOT NULL, + "tx_index" integer NOT NULL, + "index" integer NOT NULL, + "type_id" integer NOT NULL, + "raw" "bytea" NOT NULL, + "error" text NOT NULL, + CONSTRAINT "message_dead_letters_height_tx_index_index_pk" PRIMARY KEY("height","tx_index","index") +); +--> statement-breakpoint +ALTER TABLE "cosmos"."message_dead_letters" ADD CONSTRAINT "message_dead_letters_type_id_message_types_id_fk" FOREIGN KEY ("type_id") REFERENCES "cosmos"."message_types"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "message_dead_letters_type_id_idx" ON "cosmos"."message_dead_letters" USING btree ("type_id"); \ No newline at end of file diff --git a/apps/chain-indexer/drizzle/meta/0005_snapshot.json b/apps/chain-indexer/drizzle/meta/0005_snapshot.json new file mode 100644 index 0000000000..dc1dc9a49e --- /dev/null +++ b/apps/chain-indexer/drizzle/meta/0005_snapshot.json @@ -0,0 +1,1283 @@ +{ + "id": "4cf348d1-418b-4014-bbd3-4ffed0fa4e97", + "prevId": "84b7320f-f0b3-4a20-ae17-b3bcef49c056", + "version": "7", + "dialect": "postgresql", + "tables": { + "cosmos.account_balances": { + "name": "account_balances", + "schema": "cosmos", + "columns": { + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "amount": { + "name": "amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "account_balances_account_id_accounts_id_fk": { + "name": "account_balances_account_id_accounts_id_fk", + "tableFrom": "account_balances", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_balances_account_id_denom_pk": { + "name": "account_balances_account_id_denom_pk", + "columns": [ + "account_id", + "denom" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.account_txs": { + "name": "account_txs", + "schema": "cosmos", + "columns": { + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "account_tx_role", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "account_txs_account_id_accounts_id_fk": { + "name": "account_txs_account_id_accounts_id_fk", + "tableFrom": "account_txs", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_txs_account_id_height_tx_index_role_pk": { + "name": "account_txs_account_id_height_tx_index_role_pk", + "columns": [ + "account_id", + "height", + "tx_index", + "role" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.accounts": { + "name": "accounts", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_number": { + "name": "account_number", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "account_type": { + "name": "account_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_module_account": { + "name": "is_module_account", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": { + "accounts_address_idx": { + "name": "accounts_address_idx", + "columns": [ + { + "expression": "address", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.balance_changes": { + "name": "balance_changes", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delta": { + "name": "delta", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "balance_after": { + "name": "balance_after", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "balance_change_reason", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "event_index": { + "name": "event_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "counterparty_account_id": { + "name": "counterparty_account_id", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "balance_changes_account_denom_height_idx": { + "name": "balance_changes_account_denom_height_idx", + "columns": [ + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "denom", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "balance_changes_height_event_index_idx": { + "name": "balance_changes_height_event_index_idx", + "columns": [ + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_index", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "balance_changes_account_id_accounts_id_fk": { + "name": "balance_changes_account_id_accounts_id_fk", + "tableFrom": "balance_changes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "balance_changes_counterparty_account_id_accounts_id_fk": { + "name": "balance_changes_counterparty_account_id_accounts_id_fk", + "tableFrom": "balance_changes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "counterparty_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.blocks": { + "name": "blocks", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "datetime": { + "name": "datetime", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "parent_hash": { + "name": "parent_hash", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "proposer_address": { + "name": "proposer_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tx_count": { + "name": "tx_count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.delegations": { + "name": "delegations", + "schema": "cosmos", + "columns": { + "delegator_account_id": { + "name": "delegator_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "validator_operator_address": { + "name": "validator_operator_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "shares": { + "name": "shares", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "delegations_delegator_account_id_accounts_id_fk": { + "name": "delegations_delegator_account_id_accounts_id_fk", + "tableFrom": "delegations", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "delegator_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "delegations_delegator_account_id_validator_operator_address_pk": { + "name": "delegations_delegator_account_id_validator_operator_address_pk", + "columns": [ + "delegator_account_id", + "validator_operator_address" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.indexer_state": { + "name": "indexer_state", + "schema": "", + "columns": { + "stream": { + "name": "stream", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "last_height": { + "name": "last_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.message_dead_letters": { + "name": "message_dead_letters", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type_id": { + "name": "type_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "raw": { + "name": "raw", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "message_dead_letters_type_id_idx": { + "name": "message_dead_letters_type_id_idx", + "columns": [ + { + "expression": "type_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "message_dead_letters_type_id_message_types_id_fk": { + "name": "message_dead_letters_type_id_message_types_id_fk", + "tableFrom": "message_dead_letters", + "tableTo": "message_types", + "schemaTo": "cosmos", + "columnsFrom": [ + "type_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "message_dead_letters_height_tx_index_index_pk": { + "name": "message_dead_letters_height_tx_index_index_pk", + "columns": [ + "height", + "tx_index", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.message_types": { + "name": "message_types", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "message_types_type_idx": { + "name": "message_types_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.messages": { + "name": "messages", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type_id": { + "name": "type_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "body": { + "name": "body", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "messages_type_id_idx": { + "name": "messages_type_id_idx", + "columns": [ + { + "expression": "type_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "messages_type_id_message_types_id_fk": { + "name": "messages_type_id_message_types_id_fk", + "tableFrom": "messages", + "tableTo": "message_types", + "schemaTo": "cosmos", + "columnsFrom": [ + "type_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messages_height_tx_index_index_pk": { + "name": "messages_height_tx_index_index_pk", + "columns": [ + "height", + "tx_index", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.proposal_deposits": { + "name": "proposal_deposits", + "schema": "cosmos", + "columns": { + "proposal_id": { + "name": "proposal_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "depositor_account_id": { + "name": "depositor_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "amount": { + "name": "amount", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "proposal_deposits_depositor_account_id_accounts_id_fk": { + "name": "proposal_deposits_depositor_account_id_accounts_id_fk", + "tableFrom": "proposal_deposits", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "depositor_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "proposal_deposits_proposal_id_depositor_account_id_height_pk": { + "name": "proposal_deposits_proposal_id_depositor_account_id_height_pk", + "columns": [ + "proposal_id", + "depositor_account_id", + "height" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.proposal_votes": { + "name": "proposal_votes", + "schema": "cosmos", + "columns": { + "proposal_id": { + "name": "proposal_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "voter_account_id": { + "name": "voter_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "options": { + "name": "options", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "proposal_votes_voter_account_id_accounts_id_fk": { + "name": "proposal_votes_voter_account_id_accounts_id_fk", + "tableFrom": "proposal_votes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "voter_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "proposal_votes_proposal_id_voter_account_id_pk": { + "name": "proposal_votes_proposal_id_voter_account_id_pk", + "columns": [ + "proposal_id", + "voter_account_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.proposals": { + "name": "proposals", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "proposer_account_id": { + "name": "proposer_account_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "messages": { + "name": "messages", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "proposal_status", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + }, + "submit_time": { + "name": "submit_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deposit_end_time": { + "name": "deposit_end_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "voting_start_time": { + "name": "voting_start_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "voting_end_time": { + "name": "voting_end_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "total_deposit": { + "name": "total_deposit", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "final_tally_yes": { + "name": "final_tally_yes", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "final_tally_abstain": { + "name": "final_tally_abstain", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "final_tally_no": { + "name": "final_tally_no", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "final_tally_no_with_veto": { + "name": "final_tally_no_with_veto", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "submit_height": { + "name": "submit_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "proposals_proposer_account_id_accounts_id_fk": { + "name": "proposals_proposer_account_id_accounts_id_fk", + "tableFrom": "proposals", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "proposer_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.transactions": { + "name": "transactions", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "code": { + "name": "code", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "gas_used": { + "name": "gas_used", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gas_wanted": { + "name": "gas_wanted", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "fee": { + "name": "fee", + "type": "jsonb", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "transactions_hash_idx": { + "name": "transactions_hash_idx", + "columns": [ + { + "expression": "hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "transactions_height_index_pk": { + "name": "transactions_height_index_pk", + "columns": [ + "height", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.unbonding_delegations": { + "name": "unbonding_delegations", + "schema": "cosmos", + "columns": { + "delegator_account_id": { + "name": "delegator_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "validator_operator_address": { + "name": "validator_operator_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "creation_height": { + "name": "creation_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "completion_time": { + "name": "completion_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "initial_balance": { + "name": "initial_balance", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "balance": { + "name": "balance", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "unbonding_delegations_delegator_account_id_accounts_id_fk": { + "name": "unbonding_delegations_delegator_account_id_accounts_id_fk", + "tableFrom": "unbonding_delegations", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "delegator_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "unbonding_delegations_delegator_account_id_validator_operator_address_creation_height_pk": { + "name": "unbonding_delegations_delegator_account_id_validator_operator_address_creation_height_pk", + "columns": [ + "delegator_account_id", + "validator_operator_address", + "creation_height" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.validators": { + "name": "validators", + "schema": "cosmos", + "columns": { + "operator_address": { + "name": "operator_address", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_address": { + "name": "account_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "hex_address": { + "name": "hex_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "moniker": { + "name": "moniker", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "identity": { + "name": "identity", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "details": { + "name": "details", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "security_contact": { + "name": "security_contact", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commission_rate": { + "name": "commission_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "commission_max_rate": { + "name": "commission_max_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "commission_max_change_rate": { + "name": "commission_max_change_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "min_self_delegation": { + "name": "min_self_delegation", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "jailed": { + "name": "jailed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "status": { + "name": "status", + "type": "validator_status", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": false + }, + "tokens": { + "name": "tokens", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "delegator_shares": { + "name": "delegator_shares", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": false + }, + "unbonding_height": { + "name": "unbonding_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "unbonding_time": { + "name": "unbonding_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "cosmos.account_tx_role": { + "name": "account_tx_role", + "schema": "cosmos", + "values": [ + "signer", + "sender", + "receiver" + ] + }, + "cosmos.balance_change_reason": { + "name": "balance_change_reason", + "schema": "cosmos", + "values": [ + "genesis", + "transfer", + "fee", + "reward", + "commission", + "slash", + "gov", + "ibc", + "escrow", + "bme", + "mint", + "burn", + "staking" + ] + }, + "cosmos.proposal_status": { + "name": "proposal_status", + "schema": "cosmos", + "values": [ + "deposit_period", + "voting_period", + "passed", + "rejected", + "failed" + ] + }, + "cosmos.validator_status": { + "name": "validator_status", + "schema": "cosmos", + "values": [ + "unbonded", + "unbonding", + "bonded" + ] + }, + "cosmos.vote_option": { + "name": "vote_option", + "schema": "cosmos", + "values": [ + "yes", + "abstain", + "no", + "no_with_veto" + ] + } + }, + "schemas": { + "cosmos": "cosmos" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/chain-indexer/drizzle/meta/_journal.json b/apps/chain-indexer/drizzle/meta/_journal.json index c37ebe033e..7636c3e164 100644 --- a/apps/chain-indexer/drizzle/meta/_journal.json +++ b/apps/chain-indexer/drizzle/meta/_journal.json @@ -36,6 +36,13 @@ "when": 1786601805537, "tag": "0004_pretty_mister_sinister", "breakpoints": true + }, + { + "idx": 5, + "version": "7", + "when": 1786691068837, + "tag": "0005_mysterious_medusa", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/chain-indexer/env/.env.sample b/apps/chain-indexer/env/.env.sample index 2e2768a610..713e8de98d 100644 --- a/apps/chain-indexer/env/.env.sample +++ b/apps/chain-indexer/env/.env.sample @@ -12,6 +12,7 @@ ARCHIVE_BUCKET= ARCHIVE_STORAGE_API_ENDPOINT= BACKFILL_CONCURRENCY=10 BACKFILL_BATCH_SIZE=200 +BACKFILL_REPLAY=false RPC_TIMEOUT_MS=15000 MESSAGE_BODY_MAX_BYTES=65536 PORT=3092 diff --git a/apps/chain-indexer/package.json b/apps/chain-indexer/package.json index c3e9545c41..f1f809fd74 100644 --- a/apps/chain-indexer/package.json +++ b/apps/chain-indexer/package.json @@ -20,6 +20,7 @@ "test:unit": "vitest run --project unit" }, "dependencies": { + "@akashnetwork/akash-api": "1.4.3", "@akashnetwork/chain-sdk": "1.0.0-alpha.41", "@akashnetwork/env-loader": "*", "@akashnetwork/instrumentation": "*", diff --git a/apps/chain-indexer/src/config/env.config.spec.ts b/apps/chain-indexer/src/config/env.config.spec.ts index 5edc431e76..00797c7d87 100644 --- a/apps/chain-indexer/src/config/env.config.spec.ts +++ b/apps/chain-indexer/src/config/env.config.spec.ts @@ -50,6 +50,22 @@ describe("envSchema", () => { expect(config.BACKFILL_CONCURRENCY).toBe(10); expect(config.BACKFILL_BATCH_SIZE).toBe(200); }); + + it("defaults BACKFILL_REPLAY to false and treats an empty value as absent", () => { + const config = setup({ INDEXER_ROLE: "backfill", BACKFILL_FROM_HEIGHT: "100", BACKFILL_TO_HEIGHT: "200", BACKFILL_REPLAY: "" }); + + expect(config.BACKFILL_REPLAY).toBe(false); + }); + + it("parses BACKFILL_REPLAY=true as a boolean", () => { + const config = setup({ INDEXER_ROLE: "backfill", BACKFILL_FROM_HEIGHT: "100", BACKFILL_TO_HEIGHT: "200", BACKFILL_REPLAY: "true" }); + + expect(config.BACKFILL_REPLAY).toBe(true); + }); + + it("rejects a non-boolean BACKFILL_REPLAY", () => { + expect(() => setup({ INDEXER_ROLE: "backfill", BACKFILL_FROM_HEIGHT: "100", BACKFILL_TO_HEIGHT: "200", BACKFILL_REPLAY: "yes" })).toThrow(); + }); }); it("treats an empty ARCHIVE_BUCKET as absent", () => { diff --git a/apps/chain-indexer/src/config/env.config.ts b/apps/chain-indexer/src/config/env.config.ts index 47f6679afd..11815cea45 100644 --- a/apps/chain-indexer/src/config/env.config.ts +++ b/apps/chain-indexer/src/config/env.config.ts @@ -42,6 +42,12 @@ const rawEnvSchema = z.object({ BACKFILL_CONCURRENCY: z.number({ coerce: true }).int().min(1).max(64).default(10), /** How many blocks the backfill commits per Postgres transaction. */ BACKFILL_BATCH_SIZE: z.number({ coerce: true }).int().min(1).max(1_000).default(200), + /** + * Replays the range from BACKFILL_FROM_HEIGHT even when its checkpoint says complete. A replay + * fills message bodies that were null (e.g. dead-lettered types registered since) and clears + * healed dead letters; already-decoded rows are left untouched. + */ + BACKFILL_REPLAY: z.preprocess(emptyStringAsUndefined, z.enum(["true", "false"]).default("false")).transform(value => value === "true"), /** GCS bucket for the raw block archive. Unset disables archiving entirely (sync skips appends, backfill reads straight from RPC). */ ARCHIVE_BUCKET: z.preprocess(emptyStringAsUndefined, z.string().optional()), /** diff --git a/apps/chain-indexer/src/db/schema.spec.ts b/apps/chain-indexer/src/db/schema.spec.ts index a1e50d5063..562511bb2d 100644 --- a/apps/chain-indexer/src/db/schema.spec.ts +++ b/apps/chain-indexer/src/db/schema.spec.ts @@ -1,7 +1,19 @@ import { getTableConfig } from "drizzle-orm/pg-core"; import { describe, expect, it } from "vitest"; -import { AccountBalances, Accounts, AccountTxs, BalanceChanges, Delegations, ProposalDeposits, Proposals, ProposalVotes, UnbondingDelegations, Validators } from "@src/db/schema"; +import { + AccountBalances, + Accounts, + AccountTxs, + BalanceChanges, + Delegations, + MessageDeadLetters, + ProposalDeposits, + Proposals, + ProposalVotes, + UnbondingDelegations, + Validators +} from "@src/db/schema"; describe("cosmos genesis schema", () => { it("interns accounts under a unique address index", () => { @@ -63,7 +75,9 @@ describe("cosmos genesis schema", () => { it("enriches validators with snapshot-sourced bond state", () => { const config = getTableConfig(Validators); - expect(config.columns.map(column => column.name)).toEqual(expect.arrayContaining(["jailed", "status", "tokens", "delegator_shares", "unbonding_height", "unbonding_time"])); + expect(config.columns.map(column => column.name)).toEqual( + expect.arrayContaining(["jailed", "status", "tokens", "delegator_shares", "unbonding_height", "unbonding_time"]) + ); }); it("keys unbonding delegations by delegator, validator and creation height", () => { @@ -95,4 +109,14 @@ describe("cosmos genesis schema", () => { expect(config.primaryKeys[0].columns.map(column => column.name)).toEqual(["proposal_id", "depositor_account_id", "height"]); }); + + it("keys message dead letters like messages and keeps the raw bytes with the error", () => { + const config = getTableConfig(MessageDeadLetters); + + expect(config.name).toBe("message_dead_letters"); + expect(config.primaryKeys[0].columns.map(column => column.name)).toEqual(["height", "tx_index", "index"]); + expect(config.columns.map(column => column.name).sort()).toEqual(["error", "height", "index", "raw", "tx_index", "type_id"]); + expect(config.foreignKeys).toHaveLength(1); + expect(config.foreignKeys[0].reference().foreignColumns[0].name).toBe("id"); + }); }); diff --git a/apps/chain-indexer/src/db/schema.ts b/apps/chain-indexer/src/db/schema.ts index 1b87720e2e..0ee2652e62 100644 --- a/apps/chain-indexer/src/db/schema.ts +++ b/apps/chain-indexer/src/db/schema.ts @@ -70,6 +70,28 @@ export const Messages = cosmosSchema.table( t => [primaryKey({ columns: [t.height, t.txIndex, t.index] }), index("messages_type_id_idx").on(t.typeId)] ); +/** + * Messages whose body failed to decode keep their raw bytes and error here, so registering the + * type later and replaying the range can heal the null body. Re-committing a height clears its + * rows first, and a new dead-letter row is only inserted when the matching message body is still + * null, so a clean replay (or a later writer that already decoded the message) leaves no stale + * rows behind. + */ +export const MessageDeadLetters = cosmosSchema.table( + "message_dead_letters", + { + height: bigint("height", { mode: "number" }).notNull(), + txIndex: integer("tx_index").notNull(), + index: integer("index").notNull(), + typeId: integer("type_id") + .notNull() + .references(() => MessageTypes.id), + raw: bytea("raw").notNull(), + error: text("error").notNull() + }, + t => [primaryKey({ columns: [t.height, t.txIndex, t.index] }), index("message_dead_letters_type_id_idx").on(t.typeId)] +); + export const IndexerState = pgTable("indexer_state", { stream: text("stream").primaryKey(), lastHeight: bigint("last_height", { mode: "number" }).notNull(), diff --git a/apps/chain-indexer/src/db/sql-excluded.ts b/apps/chain-indexer/src/db/sql-excluded.ts new file mode 100644 index 0000000000..6d0f223b6c --- /dev/null +++ b/apps/chain-indexer/src/db/sql-excluded.ts @@ -0,0 +1,7 @@ +import type { SQL } from "drizzle-orm"; +import { sql } from "drizzle-orm"; + +/** References the proposed row's column inside an ON CONFLICT DO UPDATE clause. */ +export function sqlExcluded(column: string): SQL { + return sql.raw(`excluded.${column}`); +} diff --git a/apps/chain-indexer/src/gov/gov-writer.service.ts b/apps/chain-indexer/src/gov/gov-writer.service.ts index f956a6749d..bb1a2a3e36 100644 --- a/apps/chain-indexer/src/gov/gov-writer.service.ts +++ b/apps/chain-indexer/src/gov/gov-writer.service.ts @@ -6,6 +6,7 @@ import { INSERT_CHUNK_SIZE } from "@src/db/insert-chunk-size"; import { insertChunked } from "@src/db/insert-chunked"; import type { FeeCoin } from "@src/db/schema"; import { ProposalDeposits, Proposals, ProposalVotes } from "@src/db/schema"; +import { sqlExcluded } from "@src/db/sql-excluded"; import { sumFeeCoins } from "@src/gov/coin-total"; import type { DerivedDeposit, DerivedProposal, DerivedStatusUpdate, DerivedVote, GovChanges } from "@src/gov/gov-deriver"; import { deriveGovChanges } from "@src/gov/gov-deriver"; @@ -120,7 +121,9 @@ export class GovWriter { /** A `voting_period` promotion is conditional so it can't overwrite a terminal status; terminal updates are unconditional. */ async #applyStatusUpdates(tx: ChainTransaction, updates: DerivedStatusUpdate[]): Promise { for (const update of updates) { - const filter = update.onlyFromDepositPeriod ? and(eq(Proposals.id, update.proposalId), eq(Proposals.status, "deposit_period")) : eq(Proposals.id, update.proposalId); + const filter = update.onlyFromDepositPeriod + ? and(eq(Proposals.id, update.proposalId), eq(Proposals.status, "deposit_period")) + : eq(Proposals.id, update.proposalId); await tx.update(Proposals).set({ status: update.status }).where(filter); } } @@ -152,7 +155,3 @@ function dedupeVotes(votes: DerivedVote[]): DerivedVote[] { } return [...byKey.values()]; } - -function sqlExcluded(column: string) { - return sql.raw(`excluded.${column}`); -} diff --git a/apps/chain-indexer/src/http-schemas/status.schema.ts b/apps/chain-indexer/src/http-schemas/status.schema.ts index 43c43b1a62..bf67cfcaed 100644 --- a/apps/chain-indexer/src/http-schemas/status.schema.ts +++ b/apps/chain-indexer/src/http-schemas/status.schema.ts @@ -10,7 +10,16 @@ export const StatusResponseSchema = z.object({ lastHeight: z.number(), updatedAt: z.string() }) - ) + ), + deadLetters: z.object({ + total: z.number(), + byType: z.array( + z.object({ + type: z.string(), + count: z.number() + }) + ) + }) }) }); diff --git a/apps/chain-indexer/src/pipeline/backfill-planner.spec.ts b/apps/chain-indexer/src/pipeline/backfill-planner.spec.ts index 68670e4045..d5b3ed0618 100644 --- a/apps/chain-indexer/src/pipeline/backfill-planner.spec.ts +++ b/apps/chain-indexer/src/pipeline/backfill-planner.spec.ts @@ -38,4 +38,24 @@ describe(planBackfill.name, () => { expect(plan).toEqual({ kind: "run", startHeight: 100, endHeight: 100 }); }); + + describe("when replaying", () => { + it("re-runs a completed range from its first height", () => { + const plan = planBackfill({ fromHeight: 100, toHeight: 200, checkpointHeight: 200, tipHeight: 1_000, replay: true }); + + expect(plan).toEqual({ kind: "run", startHeight: 100, endHeight: 200 }); + }); + + it("starts at the first height even when a checkpoint sits mid-range", () => { + const plan = planBackfill({ fromHeight: 100, toHeight: 200, checkpointHeight: 150, tipHeight: 1_000, replay: true }); + + expect(plan).toEqual({ kind: "run", startHeight: 100, endHeight: 200 }); + }); + + it("still rejects a range ending above the chain tip", () => { + const plan = planBackfill({ fromHeight: 100, toHeight: 2_000, checkpointHeight: 200, tipHeight: 1_000, replay: true }); + + expect(plan).toEqual({ kind: "invalid", reason: "BACKFILL_TO_HEIGHT 2000 is above the chain tip 1000" }); + }); + }); }); diff --git a/apps/chain-indexer/src/pipeline/backfill-planner.ts b/apps/chain-indexer/src/pipeline/backfill-planner.ts index 739265f1a0..8dc17f93ba 100644 --- a/apps/chain-indexer/src/pipeline/backfill-planner.ts +++ b/apps/chain-indexer/src/pipeline/backfill-planner.ts @@ -3,6 +3,7 @@ export interface BackfillPlanInput { toHeight: number; checkpointHeight: number | null; tipHeight: number; + replay?: boolean; } export type BackfillPlan = { kind: "run"; startHeight: number; endHeight: number } | { kind: "already-complete" } | { kind: "invalid"; reason: string }; @@ -11,9 +12,11 @@ export type BackfillPlan = { kind: "run"; startHeight: number; endHeight: number * A range above the chain tip is rejected rather than clamped: clamping would mark the range's * checkpoint complete for heights that were never indexed. Completion is checked first so a * re-run of a finished range stays a no-op even when a lagging RPC node reports a stale tip. + * A replay run ignores the checkpoint entirely and starts back at `fromHeight`; commits are + * idempotent and the checkpoint only moves forward, so replaying cannot regress anything. */ export function planBackfill(input: BackfillPlanInput): BackfillPlan { - if (input.checkpointHeight !== null && input.checkpointHeight >= input.toHeight) { + if (!input.replay && input.checkpointHeight !== null && input.checkpointHeight >= input.toHeight) { return { kind: "already-complete" }; } @@ -23,7 +26,7 @@ export function planBackfill(input: BackfillPlanInput): BackfillPlan { return { kind: "run", - startHeight: input.checkpointHeight !== null ? input.checkpointHeight + 1 : input.fromHeight, + startHeight: !input.replay && input.checkpointHeight !== null ? input.checkpointHeight + 1 : input.fromHeight, endHeight: input.toHeight }; } diff --git a/apps/chain-indexer/src/pipeline/backfill-runner.service.ts b/apps/chain-indexer/src/pipeline/backfill-runner.service.ts index 68ec3e1c1a..39a255430c 100644 --- a/apps/chain-indexer/src/pipeline/backfill-runner.service.ts +++ b/apps/chain-indexer/src/pipeline/backfill-runner.service.ts @@ -83,11 +83,12 @@ export class BackfillRunnerService { } const stream = `backfill:${fromHeight}-${toHeight}`; + const replay = this.#config.BACKFILL_REPLAY; const [checkpointHeight, tipHeight] = await Promise.all([ this.#retryTransient(() => this.#getCheckpointHeight(stream), { event: "BACKFILL_CHECKPOINT_READ_RETRY" }), this.#retryTransient(() => this.#pool.getTipHeight(), { event: "BACKFILL_TIP_FETCH_RETRY" }) ]); - const plan = planBackfill({ fromHeight, toHeight, checkpointHeight, tipHeight }); + const plan = planBackfill({ fromHeight, toHeight, checkpointHeight, tipHeight, replay }); if (plan.kind === "invalid") { this.#logger.error({ event: "BACKFILL_INVALID_RANGE", reason: plan.reason }); @@ -99,8 +100,8 @@ export class BackfillRunnerService { return true; } - await this.#seedContinuityHash(plan.startHeight, checkpointHeight !== null); - this.#logger.info({ event: "BACKFILL_STARTED", network: this.#config.NETWORK, stream, startHeight: plan.startHeight, endHeight: plan.endHeight }); + await this.#seedContinuityHash(plan.startHeight, !replay && checkpointHeight !== null); + this.#logger.info({ event: "BACKFILL_STARTED", network: this.#config.NETWORK, stream, startHeight: plan.startHeight, endHeight: plan.endHeight, replay }); this.#archive.logState(); const source = new ArchiveBlockSource({ diff --git a/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts b/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts index a88932f20f..edf0407cbd 100644 --- a/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts +++ b/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts @@ -3,13 +3,14 @@ import { PgDialect } from "drizzle-orm/pg-core"; import { describe, expect, it } from "vitest"; import { mock } from "vitest-mock-extended"; -import { AccountTxs, Blocks, IndexerState, Messages, MessageTypes } from "@src/db/schema"; +import { AccountTxs, Blocks, IndexerState, MessageDeadLetters, Messages, MessageTypes } from "@src/db/schema"; import type { GovWriter } from "@src/gov/gov-writer.service"; import type { AccountInterner } from "@src/pipeline/balance/account-interner.service"; import type { BalanceWriter } from "@src/pipeline/balance/balance-writer.service"; import { BlockCommitterService } from "@src/pipeline/block-committer.service"; -import type { DecodedBlock, DecodedEvent } from "@src/pipeline/decoded-block"; +import type { DecodedBlock, DecodedEvent, MessageDecodeFailure } from "@src/pipeline/decoded-block"; import type { ChainDatabase } from "@src/providers/db.provider"; +import type { LoggerService } from "@src/providers/logging.provider"; const MSG_SEND = "/cosmos.bank.v1beta1.MsgSend"; const BALANCE_WRITE = Symbol("balance_write"); @@ -104,6 +105,88 @@ describe(BlockCommitterService.name, () => { }); }); + describe("dead letters", () => { + const UNKNOWN_TYPE = "/akash.unknown.v1.MsgMystery"; + + it("dead-letters messages whose body failed to decode and reports them loudly", async () => { + const { committer, insertedRows, logger } = setup({ + selectResults: [[{ id: 7, type: MSG_SEND }]], + insertReturning: [{ id: 8, type: UNKNOWN_TYPE }], + messagesWithNullBody: [{ height: 10, txIndex: 0, index: 1 }] + }); + const failure = { raw: new Uint8Array([1, 2, 3]), error: "Unregistered type url: /akash.unknown.v1.MsgMystery" }; + + await committer.commit(buildBlock([MSG_SEND, { typeUrl: UNKNOWN_TYPE, decodeFailure: failure }])); + + expect(insertedRows.find(call => call.table === MessageDeadLetters)?.rows).toEqual([ + { height: 10, txIndex: 0, index: 1, typeId: 8, raw: Buffer.from([1, 2, 3]), error: failure.error } + ]); + expect(logger.error).toHaveBeenCalledTimes(1); + expect(logger.error).toHaveBeenCalledWith({ + event: "MESSAGES_DEAD_LETTERED", + stream: "sync", + count: 1, + byType: { [UNKNOWN_TYPE]: 1 }, + fromHeight: 10, + toHeight: 10 + }); + }); + + it("does not re-insert a dead letter when another writer already populated the body", async () => { + const { committer, insertedRows, logger } = setup({ + selectResults: [[{ id: 7, type: MSG_SEND }]], + insertReturning: [{ id: 8, type: UNKNOWN_TYPE }], + messagesWithNullBody: [] + }); + const failure = { raw: new Uint8Array([1, 2, 3]), error: "Unregistered type url: /akash.unknown.v1.MsgMystery" }; + + await committer.commit(buildBlock([MSG_SEND, { typeUrl: UNKNOWN_TYPE, decodeFailure: failure }])); + + expect(insertedRows.find(call => call.table === MessageDeadLetters)).toBeUndefined(); + expect(logger.error).not.toHaveBeenCalled(); + }); + + it("dead-letters only the messages whose body is still null when the batch is mixed", async () => { + const { committer, insertedRows, logger } = setup({ + selectResults: [[{ id: 7, type: MSG_SEND }]], + insertReturning: [{ id: 8, type: UNKNOWN_TYPE }], + messagesWithNullBody: [{ height: 10, txIndex: 0, index: 2 }] + }); + const healed = { raw: new Uint8Array([1]), error: "Unregistered type url: /akash.unknown.v1.MsgMystery" }; + const stillUnknown = { raw: new Uint8Array([2]), error: "Unregistered type url: /akash.unknown.v1.MsgMystery" }; + + await committer.commit(buildBlock([MSG_SEND, { typeUrl: UNKNOWN_TYPE, decodeFailure: healed }, { typeUrl: UNKNOWN_TYPE, decodeFailure: stillUnknown }])); + + expect(insertedRows.find(call => call.table === MessageDeadLetters)?.rows).toEqual([ + { height: 10, txIndex: 0, index: 2, typeId: 8, raw: Buffer.from([2]), error: stillUnknown.error } + ]); + expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ event: "MESSAGES_DEAD_LETTERED", count: 1, byType: { [UNKNOWN_TYPE]: 1 } })); + }); + + it("clears the batch's height range of dead letters so a clean replay heals them", async () => { + const { committer, insertedRows, deletions, logger } = setup({ selectResults: [[{ id: 7, type: MSG_SEND }]] }); + + await committer.commitBatch([buildBlock([MSG_SEND], 10), buildBlock([MSG_SEND], 11), buildBlock([MSG_SEND], 12)], { stream: "backfill:10-12" }); + + const deletion = deletions.find(call => call.table === MessageDeadLetters); + const rendered = new PgDialect().sqlToQuery(deletion?.where as SQL); + expect(rendered.sql).toBe('"cosmos"."message_dead_letters"."height" between $1 and $2'); + expect(rendered.params).toEqual([10, 12]); + expect(insertedRows.find(call => call.table === MessageDeadLetters)).toBeUndefined(); + expect(logger.error).not.toHaveBeenCalled(); + }); + + it("heals null message bodies on conflict without touching decoded ones", async () => { + const { committer, conflictUpdates } = setup({ selectResults: [[{ id: 7, type: MSG_SEND }]] }); + + await committer.commit(buildBlock([MSG_SEND])); + + const messagesUpsert = conflictUpdates.find(call => call.table === Messages); + expect(new PgDialect().sqlToQuery(messagesUpsert?.config.set.body as SQL).sql).toBe("excluded.body"); + expect(new PgDialect().sqlToQuery(messagesUpsert?.config.setWhere as SQL).sql).toBe('"cosmos"."messages"."body" IS NULL AND excluded.body IS NOT NULL'); + }); + }); + describe("balance ledger and activity log", () => { it("writes balances then the activity log inside the transaction, after messages and before the checkpoint", async () => { const { committer, insertedRows } = setup({ selectResults: [[{ id: 7, type: MSG_SEND }]] }); @@ -155,13 +238,22 @@ describe(BlockCommitterService.name, () => { }); }); - function setup(input?: { selectResults?: Array>; insertReturning?: Array<{ id: number; type: string }> }) { + function setup(input?: { + selectResults?: Array>; + insertReturning?: Array<{ id: number; type: string }>; + messagesWithNullBody?: Array<{ height: number; txIndex: number; index: number }>; + }) { const selectResults = [...(input?.selectResults ?? [[]])]; const insertedRows: Array<{ table: unknown; rows: unknown }> = []; - const conflictUpdates: Array<{ table: unknown; config: { set: unknown } }> = []; + const conflictUpdates: Array<{ table: unknown; config: { set: Record; setWhere?: unknown } }> = []; + const deletions: Array<{ table: unknown; where: unknown }> = []; const dbFake = { - select: () => ({ from: () => ({ where: () => Promise.resolve(selectResults.shift() ?? []) }) }), + select: () => ({ + from: (table: unknown) => ({ + where: () => Promise.resolve(table === Messages ? input?.messagesWithNullBody ?? [] : selectResults.shift() ?? []) + }) + }), insert: (table: unknown) => ({ values: (rows: unknown) => { insertedRows.push({ table, rows }); @@ -170,13 +262,19 @@ describe(BlockCommitterService.name, () => { Object.assign(Promise.resolve(), { returning: () => Promise.resolve(input?.insertReturning ?? []) }), - onConflictDoUpdate: (config: { set: unknown }) => { + onConflictDoUpdate: (config: { set: Record; setWhere?: unknown }) => { conflictUpdates.push({ table, config }); return Promise.resolve(); } }; } }), + delete: (table: unknown) => ({ + where: (where: unknown) => { + deletions.push({ table, where }); + return Promise.resolve(); + } + }), transaction: (callback: (tx: unknown) => Promise) => callback(dbFake) }; @@ -189,12 +287,13 @@ describe(BlockCommitterService.name, () => { }); const govWriter = mock(); - const committer = new BlockCommitterService(dbFake as unknown as ChainDatabase, interner, balanceWriter, govWriter); - return { committer, insertedRows, conflictUpdates, interner, balanceWriter, govWriter }; + const logger = mock(); + const committer = new BlockCommitterService(dbFake as unknown as ChainDatabase, interner, balanceWriter, govWriter, logger); + return { committer, insertedRows, conflictUpdates, deletions, interner, balanceWriter, govWriter, logger }; } function buildBlock( - typeUrls: string[], + typeUrls: Array, height = 10, tx?: { events?: DecodedEvent[]; signerAddresses?: string[]; blockEvents?: DecodedEvent[] } ): DecodedBlock { @@ -212,7 +311,11 @@ describe(BlockCommitterService.name, () => { gasUsed: 0, gasWanted: 0, fee: [], - messages: typeUrls.map((typeUrl, index) => ({ index, typeUrl, body: null })), + messages: typeUrls.map((entry, index) => + typeof entry === "string" + ? { index, typeUrl: entry, body: null } + : { index, typeUrl: entry.typeUrl, body: null, decodeFailure: entry.decodeFailure } + ), events: tx?.events ?? [], signerAddresses: tx?.signerAddresses ?? [] } diff --git a/apps/chain-indexer/src/pipeline/block-committer.service.ts b/apps/chain-indexer/src/pipeline/block-committer.service.ts index f8ed3879b5..4f438abf4e 100644 --- a/apps/chain-indexer/src/pipeline/block-committer.service.ts +++ b/apps/chain-indexer/src/pipeline/block-committer.service.ts @@ -1,8 +1,11 @@ -import { inArray, sql } from "drizzle-orm"; +import { and, between, inArray, isNull, sql } from "drizzle-orm"; +import chunk from "lodash/chunk"; import { inject, singleton } from "tsyringe"; +import { INSERT_CHUNK_SIZE } from "@src/db/insert-chunk-size"; import { insertChunked } from "@src/db/insert-chunked"; -import { AccountTxs, Blocks, IndexerState, Messages, MessageTypes, Transactions } from "@src/db/schema"; +import { AccountTxs, Blocks, IndexerState, MessageDeadLetters, Messages, MessageTypes, Transactions } from "@src/db/schema"; +import { sqlExcluded } from "@src/db/sql-excluded"; import { GovWriter } from "@src/gov/gov-writer.service"; import { AccountInterner } from "@src/pipeline/balance/account-interner.service"; import type { DerivedAccountTx } from "@src/pipeline/balance/account-tx-deriver"; @@ -12,18 +15,34 @@ import { deriveBalanceChanges } from "@src/pipeline/balance/balance-deriver"; import type { ResolvedBalanceChange } from "@src/pipeline/balance/balance-writer.service"; import { BalanceWriter } from "@src/pipeline/balance/balance-writer.service"; import { buildModuleAddressRegistry } from "@src/pipeline/balance/module-address-registry"; -import type { DecodedBlock } from "@src/pipeline/decoded-block"; -import type { ChainDatabase } from "@src/providers/db.provider"; +import type { DecodedBlock, MessageDecodeFailure } from "@src/pipeline/decoded-block"; +import type { ChainDatabase, ChainTransaction } from "@src/providers/db.provider"; import { CHAIN_DB } from "@src/providers/db.provider"; +import { LoggerService } from "@src/providers/logging.provider"; export const SYNC_STREAM = "sync"; +function countByTypeUrl(messages: ReadonlyArray<{ typeUrl: string }>): Record { + const counts: Record = {}; + + for (const message of messages) { + counts[message.typeUrl] = (counts[message.typeUrl] ?? 0) + 1; + } + + return counts; +} + +function messageCoordKey(row: { height: number; txIndex: number; index: number }): string { + return `${row.height}:${row.txIndex}:${row.index}`; +} + @singleton() export class BlockCommitterService { readonly #db: ChainDatabase; readonly #interner: AccountInterner; readonly #balanceWriter: BalanceWriter; readonly #govWriter: GovWriter; + readonly #logger: LoggerService; readonly #moduleRegistry = buildModuleAddressRegistry(); readonly #typeIds = new Map(); @@ -31,12 +50,15 @@ export class BlockCommitterService { @inject(CHAIN_DB) db: ChainDatabase, @inject(AccountInterner) interner: AccountInterner, @inject(BalanceWriter) balanceWriter: BalanceWriter, - @inject(GovWriter) govWriter: GovWriter + @inject(GovWriter) govWriter: GovWriter, + @inject(LoggerService) logger: LoggerService ) { this.#db = db; this.#interner = interner; this.#balanceWriter = balanceWriter; this.#govWriter = govWriter; + this.#logger = logger; + this.#logger.setContext("COMMITTER"); } async commit(block: DecodedBlock): Promise { @@ -90,6 +112,16 @@ export class BlockCommitterService { ) ); + const deadLetteredMessages = blocks.flatMap(block => + block.transactions.flatMap(tx => + tx.messages.flatMap(message => + message.decodeFailure + ? [{ height: block.height, txIndex: tx.index, index: message.index, typeUrl: message.typeUrl, failure: message.decodeFailure }] + : [] + ) + ) + ); + const balanceChanges = blocks.flatMap(block => deriveBalanceChanges(block, this.#moduleRegistry)); const accountTxs = blocks.flatMap(block => deriveAccountTxs(block)); const accountIds = await this.#internAccounts(balanceChanges, accountTxs); @@ -98,10 +130,11 @@ export class BlockCommitterService { const lastHeight = blocks[blocks.length - 1].height; - await this.#db.transaction(async tx => { + const persistedDeadLetters = await this.#db.transaction(async tx => { await insertChunked(tx, Blocks, blockRows); await insertChunked(tx, Transactions, transactionRows); - await insertChunked(tx, Messages, messageRows); + await this.#upsertMessages(tx, messageRows); + const persisted = await this.#replaceDeadLetters(tx, blocks[0].height, lastHeight, deadLetteredMessages, typeIds); await this.#balanceWriter.write(tx, balanceIntents); await insertChunked(tx, AccountTxs, accountTxRows); @@ -114,7 +147,85 @@ export class BlockCommitterService { target: IndexerState.stream, set: { lastHeight: sql`GREATEST(${IndexerState.lastHeight}, EXCLUDED.last_height)`, updatedAt: new Date() } }); + + return persisted; }); + + if (persistedDeadLetters.length > 0) { + this.#logger.error({ + event: "MESSAGES_DEAD_LETTERED", + stream: options.stream, + count: persistedDeadLetters.length, + byType: countByTypeUrl(persistedDeadLetters), + fromHeight: blocks[0].height, + toHeight: lastHeight + }); + } + } + + /** + * Range-delete, then insert only for messages whose body is still null after the upsert. A writer + * with a stale type catalog can still fail to decode a message another writer already healed; without + * this check it would put a phantom dead-letter row back and fire MESSAGES_DEAD_LETTERED. + */ + async #replaceDeadLetters( + tx: ChainTransaction, + fromHeight: number, + toHeight: number, + deadLetteredMessages: ReadonlyArray<{ + height: number; + txIndex: number; + index: number; + typeUrl: string; + failure: MessageDecodeFailure; + }>, + typeIds: Map + ): Promise { + await tx.delete(MessageDeadLetters).where(between(MessageDeadLetters.height, fromHeight, toHeight)); + + if (deadLetteredMessages.length === 0) { + return []; + } + + const nullBodies = await tx + .select({ height: Messages.height, txIndex: Messages.txIndex, index: Messages.index }) + .from(Messages) + .where(and(between(Messages.height, fromHeight, toHeight), isNull(Messages.body))); + const nullKeys = new Set(nullBodies.map(row => messageCoordKey(row))); + const persisted = deadLetteredMessages.filter(message => nullKeys.has(messageCoordKey(message))); + + await insertChunked( + tx, + MessageDeadLetters, + persisted.map(message => ({ + height: message.height, + txIndex: message.txIndex, + index: message.index, + typeId: typeIds.get(message.typeUrl) as number, + raw: Buffer.from(message.failure.raw), + error: message.failure.error + })) + ); + + return persisted; + } + + /** + * Conflicting rows only get their body updated when it was null and the new decode produced one, + * so replaying a range after registering a previously unknown type heals the dead-lettered rows + * while normal re-commits stay write-free. + */ + async #upsertMessages(tx: ChainTransaction, rows: (typeof Messages.$inferInsert)[]): Promise { + for (const rowChunk of chunk(rows, INSERT_CHUNK_SIZE)) { + await tx + .insert(Messages) + .values(rowChunk) + .onConflictDoUpdate({ + target: [Messages.height, Messages.txIndex, Messages.index], + set: { body: sqlExcluded("body") }, + setWhere: sql`${Messages.body} IS NULL AND excluded.body IS NOT NULL` + }); + } } /** The checkpoint advances to the batch's last height, which is only correct when the batch has no gaps or reordering. */ diff --git a/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts b/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts index e5bee1a834..bb47b9234f 100644 --- a/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts +++ b/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts @@ -44,23 +44,51 @@ describe(BlockDecoderService.name, () => { }); }); - it("stores a null body for message types missing from the registry", () => { + it("marks message types missing from the registry as decode failures with their raw bytes", () => { const { decoder } = setup(); const rawTx = buildMsgSendTx("/akash.unknown.v1.MsgMystery"); const decoded = decoder.decode(buildBlock({ txs: [rawTx.toString("base64")] }), buildBlockResults([{ code: 0, gas_used: "0", gas_wanted: "0" }])); - expect(decoded.transactions[0].messages[0].typeUrl).toBe("/akash.unknown.v1.MsgMystery"); - expect(decoded.transactions[0].messages[0].body).toBeNull(); + const [message] = decoded.transactions[0].messages; + expect(message.typeUrl).toBe("/akash.unknown.v1.MsgMystery"); + expect(message.body).toBeNull(); + expect(message.decodeFailure?.error).toContain("Unregistered type url"); + expect(message.decodeFailure?.raw).toEqual(encodeMsgSendValue()); + }); + + it("marks registered message types with undecodable bytes as decode failures", () => { + const { decoder } = setup(); + const corruptBytes = new Uint8Array([0xff, 0xff, 0xff, 0xff]); + const rawTx = buildMsgSendTx("/cosmos.bank.v1beta1.MsgSend", corruptBytes); + + const decoded = decoder.decode(buildBlock({ txs: [rawTx.toString("base64")] }), buildBlockResults([{ code: 0, gas_used: "0", gas_wanted: "0" }])); + + const [message] = decoded.transactions[0].messages; + expect(message.body).toBeNull(); + expect(message.decodeFailure?.raw).toEqual(corruptBytes); + expect(message.decodeFailure?.error).toBeTruthy(); }); - it("stores a null body when the decoded message exceeds the size cap", () => { + it("skips ignored message types without marking a decode failure", () => { + const { decoder } = setup(); + const rawTx = buildMsgSendTx("/cosmwasm.wasm.v1.MsgExecuteContract"); + + const decoded = decoder.decode(buildBlock({ txs: [rawTx.toString("base64")] }), buildBlockResults([{ code: 0, gas_used: "0", gas_wanted: "0" }])); + + const [message] = decoded.transactions[0].messages; + expect(message.body).toBeNull(); + expect(message.decodeFailure).toBeUndefined(); + }); + + it("stores a null body without a decode failure when the decoded message exceeds the size cap", () => { const { decoder } = setup({ maxBodyBytes: 10 }); const rawTx = buildMsgSendTx(); const decoded = decoder.decode(buildBlock({ txs: [rawTx.toString("base64")] }), buildBlockResults([{ code: 0, gas_used: "0", gas_wanted: "0" }])); expect(decoded.transactions[0].messages[0].body).toBeNull(); + expect(decoded.transactions[0].messages[0].decodeFailure).toBeUndefined(); }); it("throws when the tx results count does not match the block txs", () => { @@ -201,14 +229,16 @@ describe(BlockDecoderService.name, () => { return { type, attributes: Object.entries(attributes).map(([key, value]) => ({ key, value })) }; } - function buildMsgSendTx(typeUrl = "/cosmos.bank.v1beta1.MsgSend"): Buffer { - const message = MsgSend.encode({ + function encodeMsgSendValue(): Uint8Array { + return MsgSend.encode({ fromAddress: "akash1from", toAddress: "akash1to", amount: [{ denom: "uakt", amount: "42" }] }).finish(); + } - const bodyBytes = TxBody.encode(TxBody.fromPartial({ messages: [{ typeUrl, value: message }] })).finish(); + function buildMsgSendTx(typeUrl = "/cosmos.bank.v1beta1.MsgSend", messageValue = encodeMsgSendValue()): Buffer { + const bodyBytes = TxBody.encode(TxBody.fromPartial({ messages: [{ typeUrl, value: messageValue }] })).finish(); const authInfoBytes = AuthInfo.encode( AuthInfo.fromPartial({ fee: { amount: [{ denom: "uakt", amount: "5000" }], gasLimit: 70_000n, payer: "", granter: "" }, signerInfos: [] }) ).finish(); diff --git a/apps/chain-indexer/src/pipeline/block-decoder.service.ts b/apps/chain-indexer/src/pipeline/block-decoder.service.ts index faa2070dd0..16fc23c5bd 100644 --- a/apps/chain-indexer/src/pipeline/block-decoder.service.ts +++ b/apps/chain-indexer/src/pipeline/block-decoder.service.ts @@ -6,8 +6,9 @@ import { inject, singleton } from "tsyringe"; import type { EnvConfig } from "@src/config/env.config"; import { toCanonicalJson } from "@src/pipeline/canonical-json"; import { decodeIfBase64 } from "@src/pipeline/decode-if-base64"; -import type { DecodedBlock, DecodedEvent, DecodedMessage, DecodedTransaction } from "@src/pipeline/decoded-block"; +import type { DecodedBlock, DecodedEvent, DecodedMessage, DecodedTransaction, MessageDecodeFailure } from "@src/pipeline/decoded-block"; import { deriveSignerAddresses } from "@src/pipeline/signer-addresses"; +import { isIgnoredTypeUrl } from "@src/proto/type-catalog"; import { APP_CONFIG } from "@src/providers/app-config.provider"; import type { Registry } from "@src/providers/type-registry.provider"; import { TYPE_REGISTRY } from "@src/providers/type-registry.provider"; @@ -113,18 +114,20 @@ export class BlockDecoderService { } #decodeMessage(message: { typeUrl: string; value: Uint8Array }, index: number): DecodedMessage { - return { - index, - typeUrl: message.typeUrl, - body: this.#decodeBody(message) - }; + const { body, failure } = this.#decodeBody(message); + + return failure ? { index, typeUrl: message.typeUrl, body, decodeFailure: failure } : { index, typeUrl: message.typeUrl, body }; } - #decodeBody(message: { typeUrl: string; value: Uint8Array }): unknown | null { + #decodeBody(message: { typeUrl: string; value: Uint8Array }): { body: unknown | null; failure?: MessageDecodeFailure } { + if (isIgnoredTypeUrl(message.typeUrl)) { + return { body: null }; + } + try { - return toCanonicalJson(this.#registry.decode(message), this.#maxBodyBytes); - } catch { - return null; + return { body: toCanonicalJson(this.#registry.decode(message), this.#maxBodyBytes) }; + } catch (error) { + return { body: null, failure: { raw: message.value, error: error instanceof Error ? error.message : String(error) } }; } } } diff --git a/apps/chain-indexer/src/pipeline/decoded-block.ts b/apps/chain-indexer/src/pipeline/decoded-block.ts index 51acfb247a..b0ee06d427 100644 --- a/apps/chain-indexer/src/pipeline/decoded-block.ts +++ b/apps/chain-indexer/src/pipeline/decoded-block.ts @@ -4,6 +4,13 @@ export interface DecodedMessage { index: number; typeUrl: string; body: unknown | null; + /** Present when the body could not be decoded (unregistered type or corrupt bytes); the committer dead-letters it. Ignored and oversized messages carry a null body without a failure. */ + decodeFailure?: MessageDecodeFailure; +} + +export interface MessageDecodeFailure { + raw: Uint8Array; + error: string; } /** An ABCI event with base64-normalized attributes flattened to a key→value map. `msgIndex` links the event to its message where present. */ diff --git a/apps/chain-indexer/src/proto/akash-type-coverage.spec.ts b/apps/chain-indexer/src/proto/akash-type-coverage.spec.ts new file mode 100644 index 0000000000..ddd67e28c5 --- /dev/null +++ b/apps/chain-indexer/src/proto/akash-type-coverage.spec.ts @@ -0,0 +1,67 @@ +import fs from "node:fs"; +import { createRequire } from "node:module"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +import { AKASH_SDK_MODULES, IGNORED_TYPE_URLS, isIgnoredTypeUrl, registeredProtoTypes } from "@src/proto/type-catalog"; + +const requireFromSpec = createRequire(import.meta.url); + +describe("akash proto type coverage", () => { + it("keeps the catalog's module map in lockstep with the installed chain SDK", () => { + const discovered = discoverSdkAkashModuleNames(); + + expect( + discovered, + "the installed @akashnetwork/chain-sdk ships a different set of Akash proto modules than the catalog imports - update AKASH_SDK_MODULES in src/proto/type-catalog.ts and register or ignore the new types" + ).toEqual(Object.keys(AKASH_SDK_MODULES).sort()); + }); + + it("registers or ignores every Akash type the installed chain SDK ships", () => { + const registeredTypeUrls = new Set(registeredProtoTypes.map(([typeUrl]) => typeUrl)); + + const unhandled = collectSdkAkashTypeUrls().filter(typeUrl => !registeredTypeUrls.has(typeUrl) && !isIgnoredTypeUrl(typeUrl)); + + expect(unhandled, "register these types in src/proto/type-catalog.ts or add them to its ignore list with a documented reason").toEqual([]); + }); + + it("keeps registered types out of the ignore list", () => { + const shadowed = registeredProtoTypes.map(([typeUrl]) => typeUrl).filter(typeUrl => isIgnoredTypeUrl(typeUrl)); + + expect(shadowed, "ignored types are never decoded, so registering them is dead weight - drop one side").toEqual([]); + }); + + it("has no stale Akash entries in the exact ignore list", () => { + const sdkTypeUrls = new Set(collectSdkAkashTypeUrls()); + + const stale = [...IGNORED_TYPE_URLS].filter(typeUrl => typeUrl.startsWith("/akash.") && !sdkTypeUrls.has(typeUrl)); + + expect(stale, "these ignored Akash types no longer exist in the installed chain SDK - remove them from IGNORED_TYPE_URLS").toEqual([]); + }); + + /** + * The universe of Akash types comes from the SDK's dist folder, not from the catalog's imports, + * so a module missing from AKASH_SDK_MODULES still surfaces its types here instead of shrinking + * the check into a vacuous pass. Off-chain `index.provider.akash.*` modules are excluded by the + * basename filter: they carry `akash.`-prefixed $types but are the provider gRPC API, never tx messages. + */ + function discoverSdkAkashModuleNames(): string[] { + const protosDir = path.dirname(requireFromSpec.resolve("@akashnetwork/chain-sdk/private-types/akash.v1")); + + return fs + .readdirSync(protosDir) + .filter(file => file.startsWith("index.akash.") && !file.endsWith(".map")) + .map(file => file.replace(/^index\./, "").replace(/\.(cjs|js)$/, "")) + .sort(); + } + + function collectSdkAkashTypeUrls(): string[] { + return discoverSdkAkashModuleNames().flatMap(name => { + const module = requireFromSpec(`@akashnetwork/chain-sdk/private-types/${name}`) as object; + + return Object.values(module).flatMap(value => + value !== null && typeof value === "object" && "$type" in value && typeof value.$type === "string" ? ["/" + value.$type] : [] + ); + }); + } +}); diff --git a/apps/chain-indexer/src/proto/type-catalog.spec.ts b/apps/chain-indexer/src/proto/type-catalog.spec.ts new file mode 100644 index 0000000000..82213ec349 --- /dev/null +++ b/apps/chain-indexer/src/proto/type-catalog.spec.ts @@ -0,0 +1,40 @@ +import { MsgCreateDeployment as legacyMsgCreateDeployment } from "@akashnetwork/akash-api/v1beta4"; +import { MsgCreateDeployment as sdkMsgCreateDeployment } from "@akashnetwork/chain-sdk/private-types/akash.v1beta4"; +import { describe, expect, it } from "vitest"; + +import { isIgnoredTypeUrl, registeredProtoTypes } from "@src/proto/type-catalog"; + +describe("type catalog", () => { + it("registers the chain SDK types including the previously missing modules", () => { + const typeUrls = new Set(registeredProtoTypes.map(([typeUrl]) => typeUrl)); + + expect(typeUrls.has("/akash.deployment.v1beta4.MsgCreateDeployment")).toBe(true); + expect(typeUrls.has("/akash.oracle.v2.MsgAddPriceEntry")).toBe(true); + expect(typeUrls.has("/akash.epochs.v1beta1.EpochInfo")).toBe(true); + expect(typeUrls.has("/cosmos.bank.v1beta1.MsgSend")).toBe(true); + expect(typeUrls.has("/ibc.applications.transfer.v1.MsgTransfer")).toBe(true); + }); + + it("registers historical Akash versions from the legacy package", () => { + const typeUrls = new Set(registeredProtoTypes.map(([typeUrl]) => typeUrl)); + + expect(typeUrls.has("/akash.deployment.v1beta1.MsgCreateDeployment")).toBe(true); + expect(typeUrls.has("/akash.deployment.v1beta2.MsgCreateDeployment")).toBe(true); + expect(typeUrls.has("/akash.market.v1beta3.MsgCreateBid")).toBe(true); + expect(typeUrls.has("/akash.market.v1beta4.MsgCreateLease")).toBe(true); + }); + + it("keeps one registration per typeUrl with the chain SDK winning over the legacy package", () => { + const typeUrls = registeredProtoTypes.map(([typeUrl]) => typeUrl); + const overlapping = registeredProtoTypes.find(([typeUrl]) => typeUrl === "/akash.deployment.v1beta4.MsgCreateDeployment"); + + expect(new Set(typeUrls).size).toBe(typeUrls.length); + expect(overlapping?.[1]).toBe(sdkMsgCreateDeployment); + expect(overlapping?.[1]).not.toBe(legacyMsgCreateDeployment); + }); + + it("ignores cosmwasm type urls by prefix", () => { + expect(isIgnoredTypeUrl("/cosmwasm.wasm.v1.MsgExecuteContract")).toBe(true); + expect(isIgnoredTypeUrl("/akash.deployment.v1beta4.MsgCreateDeployment")).toBe(false); + }); +}); diff --git a/apps/chain-indexer/src/proto/type-catalog.ts b/apps/chain-indexer/src/proto/type-catalog.ts new file mode 100644 index 0000000000..0f98d88da4 --- /dev/null +++ b/apps/chain-indexer/src/proto/type-catalog.ts @@ -0,0 +1,86 @@ +import * as legacyV1beta1 from "@akashnetwork/akash-api/v1beta1"; +import * as legacyV1beta2 from "@akashnetwork/akash-api/v1beta2"; +import * as legacyV1beta3 from "@akashnetwork/akash-api/v1beta3"; +import * as legacyV1beta4 from "@akashnetwork/akash-api/v1beta4"; +import * as akashV1 from "@akashnetwork/chain-sdk/private-types/akash.v1"; +import * as akashV1beta1 from "@akashnetwork/chain-sdk/private-types/akash.v1beta1"; +import * as akashV1beta4 from "@akashnetwork/chain-sdk/private-types/akash.v1beta4"; +import * as akashV1beta5 from "@akashnetwork/chain-sdk/private-types/akash.v1beta5"; +import * as akashV2 from "@akashnetwork/chain-sdk/private-types/akash.v2"; +import * as cosmosV1 from "@akashnetwork/chain-sdk/private-types/cosmos.v1"; +import * as cosmosV1alpha1 from "@akashnetwork/chain-sdk/private-types/cosmos.v1alpha1"; +import * as cosmosV1beta1 from "@akashnetwork/chain-sdk/private-types/cosmos.v1beta1"; +import * as cosmosV2alpha1 from "@akashnetwork/chain-sdk/private-types/cosmos.v2alpha1"; +import type { GeneratedType } from "@cosmjs/proto-signing"; +import { defaultRegistryTypes as stargateDefaultRegistryTypes } from "@cosmjs/stargate"; + +/** + * Every `index.akash.*` proto module of the installed chain SDK, keyed by module basename. + * `akash-type-coverage.spec.ts` asserts this map stays in lockstep with the SDK's dist folder, + * so an SDK bump that ships a new Akash proto module cannot merge without being added here + * (registered) or listed in the ignore set. + */ +export const AKASH_SDK_MODULES: Readonly> = { + "akash.v1": akashV1, + "akash.v1beta1": akashV1beta1, + "akash.v1beta4": akashV1beta4, + "akash.v1beta5": akashV1beta5, + "akash.v2": akashV2 +}; + +/** + * Historical Akash proto versions that no longer ship in the chain SDK but still appear in + * mainnet history. Decoding them from the frozen legacy package keeps backfilled message + * bodies populated instead of dead-lettering types we already know about. + */ +const LEGACY_AKASH_MODULES: readonly object[] = [legacyV1beta1, legacyV1beta2, legacyV1beta3, legacyV1beta4]; + +const COSMOS_SDK_MODULES: readonly object[] = [cosmosV1, cosmosV1beta1, cosmosV1alpha1, cosmosV2alpha1]; + +/** + * Message families the indexer deliberately does not decode: their bodies are stored as null + * without dead-lettering. Every entry needs a reason here, and the exact-match set is checked + * against the installed SDK by `akash-type-coverage.spec.ts` so stale entries fail CI. + * + * - `/cosmwasm.`: cosmwasm runs on sandbox only and no consumer reads contract call bodies yet. + */ +export const IGNORED_TYPE_URL_PREFIXES: readonly string[] = ["/cosmwasm."]; + +export const IGNORED_TYPE_URLS: ReadonlySet = new Set(); + +export function isIgnoredTypeUrl(typeUrl: string): boolean { + return IGNORED_TYPE_URLS.has(typeUrl) || IGNORED_TYPE_URL_PREFIXES.some(prefix => typeUrl.startsWith(prefix)); +} + +function collectTypePairs(module: object): Array<[string, GeneratedType]> { + return Object.values(module).flatMap(value => + value !== null && typeof value === "object" && "$type" in value && typeof value.$type === "string" + ? [["/" + value.$type, value as unknown as GeneratedType] as [string, GeneratedType]] + : [] + ); +} + +/** First registration wins, so the chain SDK's codegen takes precedence over the legacy package where versions overlap (e.g. deployment v1beta4). */ +function dedupeFirstWins(groups: ReadonlyArray>): Array<[string, GeneratedType]> { + const byTypeUrl = new Map(); + + for (const pairs of groups) { + for (const [typeUrl, type] of pairs) { + if (!byTypeUrl.has(typeUrl)) { + byTypeUrl.set(typeUrl, type); + } + } + } + + return [...byTypeUrl.entries()]; +} + +const ibcTypes: Array<[string, GeneratedType]> = stargateDefaultRegistryTypes.filter(([type]) => type.startsWith("/ibc")); + +/** Everything the block decoder can decode, deduped by typeUrl. The registry provider turns this into the cosmjs Registry. */ +export const registeredProtoTypes: ReadonlyArray<[string, GeneratedType]> = dedupeFirstWins([ + ...Object.values(AKASH_SDK_MODULES).map(collectTypePairs), + ...COSMOS_SDK_MODULES.map(collectTypePairs), + ibcTypes, + ...LEGACY_AKASH_MODULES.map(collectTypePairs) +]); diff --git a/apps/chain-indexer/src/providers/type-registry.provider.ts b/apps/chain-indexer/src/providers/type-registry.provider.ts index 84cf7d2c1d..c3ddc056ec 100644 --- a/apps/chain-indexer/src/providers/type-registry.provider.ts +++ b/apps/chain-indexer/src/providers/type-registry.provider.ts @@ -1,33 +1,10 @@ -import * as v1 from "@akashnetwork/chain-sdk/private-types/akash.v1"; -import * as v1beta4 from "@akashnetwork/chain-sdk/private-types/akash.v1beta4"; -import * as v1beta5 from "@akashnetwork/chain-sdk/private-types/akash.v1beta5"; -import * as cosmosv1 from "@akashnetwork/chain-sdk/private-types/cosmos.v1"; -import * as cosmosv1alpha1 from "@akashnetwork/chain-sdk/private-types/cosmos.v1alpha1"; -import * as cosmosv1beta1 from "@akashnetwork/chain-sdk/private-types/cosmos.v1beta1"; -import * as cosmosv2alpha1 from "@akashnetwork/chain-sdk/private-types/cosmos.v2alpha1"; -import type { GeneratedType } from "@cosmjs/proto-signing"; import { Registry } from "@cosmjs/proto-signing"; -import { defaultRegistryTypes as stargateDefaultRegistryTypes } from "@cosmjs/stargate"; import type { InjectionToken } from "tsyringe"; import { container } from "tsyringe"; -type DefaultRegistryType = [string, GeneratedType]; -const ibcTypes: DefaultRegistryType[] = stargateDefaultRegistryTypes.filter(([type]) => type.startsWith("/ibc")); -const defaultRegistryTypes: DefaultRegistryType[] = [ - ...Object.values(cosmosv1), - ...Object.values(cosmosv1beta1), - ...Object.values(cosmosv1alpha1), - ...Object.values(cosmosv2alpha1) -] - .filter(x => x && "$type" in x) - .map(x => ["/" + x.$type, x as unknown as GeneratedType]) - .concat(ibcTypes) as DefaultRegistryType[]; +import { registeredProtoTypes } from "@src/proto/type-catalog"; -const akashTypes: ReadonlyArray<[string, GeneratedType]> = [...Object.values(v1), ...Object.values(v1beta4), ...Object.values(v1beta5)] - .filter(x => "$type" in x) - .map(x => ["/" + x.$type, x as unknown as GeneratedType]); - -const registry = new Registry([...defaultRegistryTypes, ...akashTypes]); +const registry = new Registry(registeredProtoTypes); export const TYPE_REGISTRY: InjectionToken = Symbol("TYPE_REGISTRY"); export type { Registry }; diff --git a/apps/chain-indexer/src/services/status/status.service.spec.ts b/apps/chain-indexer/src/services/status/status.service.spec.ts new file mode 100644 index 0000000000..8fa4415249 --- /dev/null +++ b/apps/chain-indexer/src/services/status/status.service.spec.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "vitest"; + +import { envSchema } from "@src/config/env.config"; +import { IndexerState } from "@src/db/schema"; +import { StatusResponseSchema } from "@src/http-schemas/status.schema"; +import type { ChainDatabase } from "@src/providers/db.provider"; +import { StatusService } from "@src/services/status/status.service"; + +describe(StatusService.name, () => { + it("returns checkpoints with dead-letter counts grouped by type", async () => { + const { service } = setup({ + checkpoints: [{ stream: "sync", lastHeight: 42, updatedAt: new Date("2026-08-14T00:00:00Z") }], + deadLetterCounts: [ + { type: "/akash.unknown.v1.MsgMystery", count: 2 }, + { type: "/cosmos.unknown.v1.MsgOther", count: 1 } + ] + }); + + const status = await service.getStatus(); + + expect(status.data.network).toBe("sandbox"); + expect(status.data.role).toBe("sync"); + expect(status.data.checkpoints).toEqual([{ stream: "sync", lastHeight: 42, updatedAt: "2026-08-14T00:00:00.000Z" }]); + expect(status.data.deadLetters).toEqual({ + total: 3, + byType: [ + { type: "/akash.unknown.v1.MsgMystery", count: 2 }, + { type: "/cosmos.unknown.v1.MsgOther", count: 1 } + ] + }); + expect(StatusResponseSchema.parse(status)).toEqual(status); + }); + + it("reports zero dead letters when the store is empty", async () => { + const { service } = setup({ checkpoints: [], deadLetterCounts: [] }); + + const status = await service.getStatus(); + + expect(status.data.deadLetters).toEqual({ total: 0, byType: [] }); + }); + + function setup(input: { + checkpoints: Array<{ stream: string; lastHeight: number; updatedAt: Date }>; + deadLetterCounts: Array<{ type: string; count: number }>; + }) { + const dbFake = { + select: () => ({ + from: (table: unknown) => + Object.assign(Promise.resolve(table === IndexerState ? input.checkpoints : []), { + innerJoin: () => ({ groupBy: () => Promise.resolve(input.deadLetterCounts) }) + }) + }) + }; + + const config = envSchema.parse({ POSTGRES_DB_URI: "postgres://unit:unit@localhost:5432/unit" }); + const service = new StatusService(dbFake as unknown as ChainDatabase, config); + return { service }; + } +}); diff --git a/apps/chain-indexer/src/services/status/status.service.ts b/apps/chain-indexer/src/services/status/status.service.ts index 82a11812de..c7f1d4d733 100644 --- a/apps/chain-indexer/src/services/status/status.service.ts +++ b/apps/chain-indexer/src/services/status/status.service.ts @@ -1,7 +1,8 @@ +import { count, eq } from "drizzle-orm"; import { inject, singleton } from "tsyringe"; import type { EnvConfig } from "@src/config/env.config"; -import { IndexerState } from "@src/db/schema"; +import { IndexerState, MessageDeadLetters, MessageTypes } from "@src/db/schema"; import type { StatusResponse } from "@src/http-schemas/status.schema"; import { APP_CONFIG } from "@src/providers/app-config.provider"; import type { ChainDatabase } from "@src/providers/db.provider"; @@ -18,7 +19,7 @@ export class StatusService { } async getStatus(): Promise { - const checkpoints = await this.#db.select().from(IndexerState); + const [checkpoints, deadLetters] = await Promise.all([this.#db.select().from(IndexerState), this.#countDeadLettersByType()]); return { data: { @@ -28,8 +29,20 @@ export class StatusService { stream: checkpoint.stream, lastHeight: checkpoint.lastHeight, updatedAt: checkpoint.updatedAt.toISOString() - })) + })), + deadLetters: { + total: deadLetters.reduce((total, row) => total + row.count, 0), + byType: deadLetters + } } }; } + + async #countDeadLettersByType(): Promise> { + return await this.#db + .select({ type: MessageTypes.type, count: count() }) + .from(MessageDeadLetters) + .innerJoin(MessageTypes, eq(MessageDeadLetters.typeId, MessageTypes.id)) + .groupBy(MessageTypes.type); + } } diff --git a/package-lock.json b/package-lock.json index 26f51b6bd1..ce10e800f9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -536,6 +536,7 @@ "version": "0.0.1", "license": "Apache-2.0", "dependencies": { + "@akashnetwork/akash-api": "1.4.3", "@akashnetwork/chain-sdk": "1.0.0-alpha.41", "@akashnetwork/env-loader": "*", "@akashnetwork/instrumentation": "*", From 30b6132ec43b38be3a9b51afed890dc050460526 Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Sun, 16 Aug 2026 20:19:53 +0530 Subject: [PATCH 09/14] feat(indexer): akash deployment and market handlers with escrow settlement (#3600) * feat(indexer): akash deployment and market handlers with escrow settlement * fix(indexer): harden akash settlement and identity parsing from review Address the CodeRabbit review on the deployment/market handlers: - settlement: guard a zero total block rate before dividing, so a degraded or orphan zero-price lease no longer aborts the block commit - settlement: bound the post-distribution dust check by absolute value and include the height, so a small negative residual can't slip past - json: asInteger now rejects negative and non-safe integers - uint64: asUint64String validates the uint64 range, rejects unsafe numbers and malformed Long halves, and normalizes digit strings - denom: look up the mapping through a Map so a denom like "constructor" degrades to unmapped instead of resolving a prototype member - schema: widen deployments.id and deployment_groups.id to bigserial with bigint foreign keys, so the guarded upserts cannot exhaust an int4 sequence; migration 0006 regenerated - writer: group child rows by deployment once in loadStates instead of re-scanning every child table per deployment Adds unit coverage for the zero-rate, orphan-lease, negative-decimal, uint64-validation, escrow string-scope, and empty-returning re-select paths. * fix(indexer): close losing bids on lease creation and tidy akash handlers From the two-axis code review of the deployment/market handlers. Behavioral fix: creating a lease matches and closes its order on-chain, so the order's other still-open bids are now closed at that height. Previously they stayed "open" until the whole deployment closed. This is bid-table accuracy only; the event timeline is unchanged. Behavior-preserving cleanups from the standards axis: - share MSG_EXEC_TYPE_URL and the exec-depth cap between the block decoder and the akash deriver (new src/pipeline/msg-exec.ts) - dispatch via normalizeDeployment ?? normalizeMarket and drop the now redundant isDeploymentTypeUrl / isMarketTypeUrl predicates - extract sumLeaseRate() for the repeated open-lease block-rate sum - extract ownerDseqKey() for the repeated owner/dseq map key in the writer - name the {gseq, oseq, bseq, provider} clump as LeaseSlot - share akashTypeUrlSet() across the deployment and market normalizers - drop the redundant NormalizedChange type alias * fix(indexer): decode legacy DecCoin amounts exactly instead of via float64 Found by verifying the deployment/market handlers against mainnet history: @akashnetwork/akash-api patches DecCoin.decode with parseInt(atomics) / 1e18, which overflows float64 precision and corrupts every v1beta1-v1beta4 DecCoin at the ~15th significant digit (a real v1beta2 bid price of 117.73952 uakt/block decoded as 117.739519999999999). All legacy proto versions share one coin module instance, so re-patching its decode with exact string math (atomics string -> decimal string) fixes stored message bodies, bid/lease prices and settlement inputs in one place. Verified against akashnet-2 archival history: with the fix, indexed prices match on-chain bids exactly, and escrow refunds/payouts reconcile with the bank transfer events emitted at lease withdraw and deployment close. --- .../drizzle/0006_next_earthquake.sql | 117 + .../drizzle/meta/0006_snapshot.json | 2158 +++++++++++++++++ apps/chain-indexer/drizzle/meta/_journal.json | 7 + apps/chain-indexer/package.json | 1 + apps/chain-indexer/src/akash/akash-changes.ts | 90 + .../src/akash/akash-deriver.spec.ts | 173 ++ apps/chain-indexer/src/akash/akash-deriver.ts | 142 ++ .../src/akash/akash-writer.service.spec.ts | 301 +++ .../src/akash/akash-writer.service.ts | 525 ++++ apps/chain-indexer/src/akash/dec.spec.ts | 107 + apps/chain-indexer/src/akash/dec.ts | 75 + apps/chain-indexer/src/akash/denom.ts | 16 + .../src/akash/deployment-reducer.spec.ts | 324 +++ .../src/akash/deployment-reducer.ts | 584 +++++ apps/chain-indexer/src/akash/json.spec.ts | 28 + apps/chain-indexer/src/akash/json.ts | 18 + .../src/akash/normalize-deployment.spec.ts | 111 + .../src/akash/normalize-deployment.ts | 115 + .../src/akash/normalize-market.spec.ts | 55 + .../src/akash/normalize-market.ts | 65 + .../chain-indexer/src/akash/resources.spec.ts | 131 + apps/chain-indexer/src/akash/resources.ts | 96 + .../src/akash/settlement.spec.ts | 124 + apps/chain-indexer/src/akash/settlement.ts | 86 + apps/chain-indexer/src/akash/uint64.spec.ts | 46 + apps/chain-indexer/src/akash/uint64.ts | 30 + apps/chain-indexer/src/db/schema.spec.ts | 64 + apps/chain-indexer/src/db/schema.ts | 197 ++ .../pipeline/block-committer.service.spec.ts | 24 +- .../src/pipeline/block-committer.service.ts | 26 +- .../pipeline/block-decoder.service.spec.ts | 54 + .../src/pipeline/block-decoder.service.ts | 41 +- apps/chain-indexer/src/pipeline/msg-exec.ts | 5 + .../proto/legacy-dec-coin-precision.spec.ts | 28 + .../src/proto/legacy-dec-coin-precision.ts | 50 + apps/chain-indexer/src/proto/type-catalog.ts | 2 + package-lock.json | 2 +- 37 files changed, 6006 insertions(+), 12 deletions(-) create mode 100644 apps/chain-indexer/drizzle/0006_next_earthquake.sql create mode 100644 apps/chain-indexer/drizzle/meta/0006_snapshot.json create mode 100644 apps/chain-indexer/src/akash/akash-changes.ts create mode 100644 apps/chain-indexer/src/akash/akash-deriver.spec.ts create mode 100644 apps/chain-indexer/src/akash/akash-deriver.ts create mode 100644 apps/chain-indexer/src/akash/akash-writer.service.spec.ts create mode 100644 apps/chain-indexer/src/akash/akash-writer.service.ts create mode 100644 apps/chain-indexer/src/akash/dec.spec.ts create mode 100644 apps/chain-indexer/src/akash/dec.ts create mode 100644 apps/chain-indexer/src/akash/denom.ts create mode 100644 apps/chain-indexer/src/akash/deployment-reducer.spec.ts create mode 100644 apps/chain-indexer/src/akash/deployment-reducer.ts create mode 100644 apps/chain-indexer/src/akash/json.spec.ts create mode 100644 apps/chain-indexer/src/akash/json.ts create mode 100644 apps/chain-indexer/src/akash/normalize-deployment.spec.ts create mode 100644 apps/chain-indexer/src/akash/normalize-deployment.ts create mode 100644 apps/chain-indexer/src/akash/normalize-market.spec.ts create mode 100644 apps/chain-indexer/src/akash/normalize-market.ts create mode 100644 apps/chain-indexer/src/akash/resources.spec.ts create mode 100644 apps/chain-indexer/src/akash/resources.ts create mode 100644 apps/chain-indexer/src/akash/settlement.spec.ts create mode 100644 apps/chain-indexer/src/akash/settlement.ts create mode 100644 apps/chain-indexer/src/akash/uint64.spec.ts create mode 100644 apps/chain-indexer/src/akash/uint64.ts create mode 100644 apps/chain-indexer/src/pipeline/msg-exec.ts create mode 100644 apps/chain-indexer/src/proto/legacy-dec-coin-precision.spec.ts create mode 100644 apps/chain-indexer/src/proto/legacy-dec-coin-precision.ts diff --git a/apps/chain-indexer/drizzle/0006_next_earthquake.sql b/apps/chain-indexer/drizzle/0006_next_earthquake.sql new file mode 100644 index 0000000000..4a978882db --- /dev/null +++ b/apps/chain-indexer/drizzle/0006_next_earthquake.sql @@ -0,0 +1,117 @@ +CREATE SCHEMA "akash"; +--> statement-breakpoint +CREATE TYPE "akash"."bid_state" AS ENUM('open', 'active', 'closed');--> statement-breakpoint +CREATE TYPE "akash"."deployment_close_reason" AS ENUM('close_message', 'overdrawn', 'close_event');--> statement-breakpoint +CREATE TYPE "akash"."deployment_event_type" AS ENUM('created', 'deposited', 'updated', 'closed', 'group_closed', 'group_paused', 'group_started', 'bid_created', 'bid_closed', 'lease_created', 'lease_closed', 'lease_withdrawn');--> statement-breakpoint +CREATE TYPE "akash"."group_state" AS ENUM('open', 'paused', 'closed');--> statement-breakpoint +CREATE TABLE "akash"."bids" ( + "deployment_id" bigint NOT NULL, + "gseq" integer NOT NULL, + "oseq" integer NOT NULL, + "bseq" integer DEFAULT 0 NOT NULL, + "provider_account_id" integer NOT NULL, + "price" numeric(38, 18) NOT NULL, + "denom" text NOT NULL, + "state" "akash"."bid_state" DEFAULT 'open' NOT NULL, + "created_height" bigint NOT NULL, + "closed_height" bigint, + CONSTRAINT "bids_deployment_id_gseq_oseq_bseq_provider_account_id_pk" PRIMARY KEY("deployment_id","gseq","oseq","bseq","provider_account_id") +); +--> statement-breakpoint +CREATE TABLE "akash"."deployment_events" ( + "deployment_id" bigint NOT NULL, + "height" bigint NOT NULL, + "ordinal" integer NOT NULL, + "tx_index" integer, + "msg_index" integer, + "type" "akash"."deployment_event_type" NOT NULL, + "details" jsonb, + CONSTRAINT "deployment_events_deployment_id_height_ordinal_pk" PRIMARY KEY("deployment_id","height","ordinal") +); +--> statement-breakpoint +CREATE TABLE "akash"."deployment_group_resources" ( + "deployment_group_id" bigint NOT NULL, + "idx" integer NOT NULL, + "count" integer NOT NULL, + "cpu_units" bigint NOT NULL, + "gpu_units" bigint NOT NULL, + "gpu_vendor" text, + "gpu_model" text, + "memory_bytes" bigint NOT NULL, + "ephemeral_storage_bytes" bigint NOT NULL, + "persistent_storage_bytes" bigint NOT NULL, + "price" numeric(38, 18) NOT NULL, + "price_denom" text NOT NULL, + CONSTRAINT "deployment_group_resources_deployment_group_id_idx_pk" PRIMARY KEY("deployment_group_id","idx") +); +--> statement-breakpoint +CREATE TABLE "akash"."deployment_groups" ( + "id" bigserial PRIMARY KEY NOT NULL, + "deployment_id" bigint NOT NULL, + "gseq" integer NOT NULL, + "state" "akash"."group_state" DEFAULT 'open' NOT NULL, + "closed_height" bigint +); +--> statement-breakpoint +CREATE TABLE "akash"."deployments" ( + "id" bigserial PRIMARY KEY NOT NULL, + "owner_account_id" integer NOT NULL, + "dseq" numeric(20, 0) NOT NULL, + "denom" text NOT NULL, + "deposit" numeric(38, 0) NOT NULL, + "balance" numeric(38, 18) NOT NULL, + "withdrawn_amount" numeric(38, 18) NOT NULL, + "block_rate" numeric(38, 18) DEFAULT '0' NOT NULL, + "last_withdraw_height" bigint, + "last_processed_height" bigint NOT NULL, + "created_height" bigint NOT NULL, + "created_at" timestamp with time zone NOT NULL, + "closed_height" bigint, + "closed_at" timestamp with time zone, + "close_reason" "akash"."deployment_close_reason", + "cpu_units" bigint NOT NULL, + "gpu_units" bigint NOT NULL, + "memory_bytes" bigint NOT NULL, + "ephemeral_storage_bytes" bigint NOT NULL, + "persistent_storage_bytes" bigint NOT NULL +); +--> statement-breakpoint +CREATE TABLE "akash"."leases" ( + "deployment_id" bigint NOT NULL, + "deployment_group_id" bigint NOT NULL, + "gseq" integer NOT NULL, + "oseq" integer NOT NULL, + "bseq" integer DEFAULT 0 NOT NULL, + "provider_account_id" integer NOT NULL, + "price" numeric(38, 18) NOT NULL, + "denom" text NOT NULL, + "balance" numeric(38, 18) DEFAULT '0' NOT NULL, + "withdrawn_amount" numeric(38, 18) DEFAULT '0' NOT NULL, + "predicted_closed_height" numeric(30, 0) NOT NULL, + "created_height" bigint NOT NULL, + "created_at" timestamp with time zone NOT NULL, + "closed_height" bigint, + "closed_at" timestamp with time zone, + "cpu_units" bigint NOT NULL, + "gpu_units" bigint NOT NULL, + "memory_bytes" bigint NOT NULL, + "ephemeral_storage_bytes" bigint NOT NULL, + "persistent_storage_bytes" bigint NOT NULL, + CONSTRAINT "leases_deployment_id_gseq_oseq_bseq_provider_account_id_pk" PRIMARY KEY("deployment_id","gseq","oseq","bseq","provider_account_id") +); +--> statement-breakpoint +ALTER TABLE "akash"."bids" ADD CONSTRAINT "bids_deployment_id_deployments_id_fk" FOREIGN KEY ("deployment_id") REFERENCES "akash"."deployments"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "akash"."bids" ADD CONSTRAINT "bids_provider_account_id_accounts_id_fk" FOREIGN KEY ("provider_account_id") REFERENCES "cosmos"."accounts"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "akash"."deployment_events" ADD CONSTRAINT "deployment_events_deployment_id_deployments_id_fk" FOREIGN KEY ("deployment_id") REFERENCES "akash"."deployments"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "akash"."deployment_group_resources" ADD CONSTRAINT "deployment_group_resources_deployment_group_id_deployment_groups_id_fk" FOREIGN KEY ("deployment_group_id") REFERENCES "akash"."deployment_groups"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "akash"."deployment_groups" ADD CONSTRAINT "deployment_groups_deployment_id_deployments_id_fk" FOREIGN KEY ("deployment_id") REFERENCES "akash"."deployments"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "akash"."deployments" ADD CONSTRAINT "deployments_owner_account_id_accounts_id_fk" FOREIGN KEY ("owner_account_id") REFERENCES "cosmos"."accounts"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "akash"."leases" ADD CONSTRAINT "leases_deployment_id_deployments_id_fk" FOREIGN KEY ("deployment_id") REFERENCES "akash"."deployments"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "akash"."leases" ADD CONSTRAINT "leases_deployment_group_id_deployment_groups_id_fk" FOREIGN KEY ("deployment_group_id") REFERENCES "akash"."deployment_groups"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "akash"."leases" ADD CONSTRAINT "leases_provider_account_id_accounts_id_fk" FOREIGN KEY ("provider_account_id") REFERENCES "cosmos"."accounts"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +CREATE UNIQUE INDEX "deployment_groups_deployment_gseq_idx" ON "akash"."deployment_groups" USING btree ("deployment_id","gseq");--> statement-breakpoint +CREATE UNIQUE INDEX "deployments_owner_dseq_idx" ON "akash"."deployments" USING btree ("owner_account_id","dseq");--> statement-breakpoint +CREATE INDEX "deployments_owner_created_idx" ON "akash"."deployments" USING btree ("owner_account_id","created_height");--> statement-breakpoint +CREATE INDEX "deployments_open_idx" ON "akash"."deployments" USING btree ("created_height") WHERE "akash"."deployments"."closed_height" IS NULL;--> statement-breakpoint +CREATE INDEX "leases_provider_idx" ON "akash"."leases" USING btree ("provider_account_id","closed_height","created_height");--> statement-breakpoint +CREATE INDEX "leases_open_idx" ON "akash"."leases" USING btree ("deployment_id") WHERE "akash"."leases"."closed_height" IS NULL; \ No newline at end of file diff --git a/apps/chain-indexer/drizzle/meta/0006_snapshot.json b/apps/chain-indexer/drizzle/meta/0006_snapshot.json new file mode 100644 index 0000000000..564f8d90b4 --- /dev/null +++ b/apps/chain-indexer/drizzle/meta/0006_snapshot.json @@ -0,0 +1,2158 @@ +{ + "id": "7703e3c3-9988-452e-ae30-44e28b863168", + "prevId": "4cf348d1-418b-4014-bbd3-4ffed0fa4e97", + "version": "7", + "dialect": "postgresql", + "tables": { + "cosmos.account_balances": { + "name": "account_balances", + "schema": "cosmos", + "columns": { + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "amount": { + "name": "amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "account_balances_account_id_accounts_id_fk": { + "name": "account_balances_account_id_accounts_id_fk", + "tableFrom": "account_balances", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_balances_account_id_denom_pk": { + "name": "account_balances_account_id_denom_pk", + "columns": [ + "account_id", + "denom" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.account_txs": { + "name": "account_txs", + "schema": "cosmos", + "columns": { + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "account_tx_role", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "account_txs_account_id_accounts_id_fk": { + "name": "account_txs_account_id_accounts_id_fk", + "tableFrom": "account_txs", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_txs_account_id_height_tx_index_role_pk": { + "name": "account_txs_account_id_height_tx_index_role_pk", + "columns": [ + "account_id", + "height", + "tx_index", + "role" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.accounts": { + "name": "accounts", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_number": { + "name": "account_number", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "account_type": { + "name": "account_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_module_account": { + "name": "is_module_account", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": { + "accounts_address_idx": { + "name": "accounts_address_idx", + "columns": [ + { + "expression": "address", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.balance_changes": { + "name": "balance_changes", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delta": { + "name": "delta", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "balance_after": { + "name": "balance_after", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "balance_change_reason", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "event_index": { + "name": "event_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "counterparty_account_id": { + "name": "counterparty_account_id", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "balance_changes_account_denom_height_idx": { + "name": "balance_changes_account_denom_height_idx", + "columns": [ + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "denom", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "balance_changes_height_event_index_idx": { + "name": "balance_changes_height_event_index_idx", + "columns": [ + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_index", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "balance_changes_account_id_accounts_id_fk": { + "name": "balance_changes_account_id_accounts_id_fk", + "tableFrom": "balance_changes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "balance_changes_counterparty_account_id_accounts_id_fk": { + "name": "balance_changes_counterparty_account_id_accounts_id_fk", + "tableFrom": "balance_changes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "counterparty_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.bids": { + "name": "bids", + "schema": "akash", + "columns": { + "deployment_id": { + "name": "deployment_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gseq": { + "name": "gseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "oseq": { + "name": "oseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "bseq": { + "name": "bseq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "provider_account_id": { + "name": "provider_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "price": { + "name": "price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "bid_state", + "typeSchema": "akash", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "created_height": { + "name": "created_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "closed_height": { + "name": "closed_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "bids_deployment_id_deployments_id_fk": { + "name": "bids_deployment_id_deployments_id_fk", + "tableFrom": "bids", + "tableTo": "deployments", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "bids_provider_account_id_accounts_id_fk": { + "name": "bids_provider_account_id_accounts_id_fk", + "tableFrom": "bids", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "provider_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "bids_deployment_id_gseq_oseq_bseq_provider_account_id_pk": { + "name": "bids_deployment_id_gseq_oseq_bseq_provider_account_id_pk", + "columns": [ + "deployment_id", + "gseq", + "oseq", + "bseq", + "provider_account_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.blocks": { + "name": "blocks", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "datetime": { + "name": "datetime", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "parent_hash": { + "name": "parent_hash", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "proposer_address": { + "name": "proposer_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tx_count": { + "name": "tx_count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.delegations": { + "name": "delegations", + "schema": "cosmos", + "columns": { + "delegator_account_id": { + "name": "delegator_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "validator_operator_address": { + "name": "validator_operator_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "shares": { + "name": "shares", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "delegations_delegator_account_id_accounts_id_fk": { + "name": "delegations_delegator_account_id_accounts_id_fk", + "tableFrom": "delegations", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "delegator_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "delegations_delegator_account_id_validator_operator_address_pk": { + "name": "delegations_delegator_account_id_validator_operator_address_pk", + "columns": [ + "delegator_account_id", + "validator_operator_address" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.deployment_events": { + "name": "deployment_events", + "schema": "akash", + "columns": { + "deployment_id": { + "name": "deployment_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ordinal": { + "name": "ordinal", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "msg_index": { + "name": "msg_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "deployment_event_type", + "typeSchema": "akash", + "primaryKey": false, + "notNull": true + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "deployment_events_deployment_id_deployments_id_fk": { + "name": "deployment_events_deployment_id_deployments_id_fk", + "tableFrom": "deployment_events", + "tableTo": "deployments", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "deployment_events_deployment_id_height_ordinal_pk": { + "name": "deployment_events_deployment_id_height_ordinal_pk", + "columns": [ + "deployment_id", + "height", + "ordinal" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.deployment_group_resources": { + "name": "deployment_group_resources", + "schema": "akash", + "columns": { + "deployment_group_id": { + "name": "deployment_group_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "idx": { + "name": "idx", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "cpu_units": { + "name": "cpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gpu_units": { + "name": "gpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gpu_vendor": { + "name": "gpu_vendor", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "gpu_model": { + "name": "gpu_model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "memory_bytes": { + "name": "memory_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ephemeral_storage_bytes": { + "name": "ephemeral_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "persistent_storage_bytes": { + "name": "persistent_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "price": { + "name": "price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "price_denom": { + "name": "price_denom", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "deployment_group_resources_deployment_group_id_deployment_groups_id_fk": { + "name": "deployment_group_resources_deployment_group_id_deployment_groups_id_fk", + "tableFrom": "deployment_group_resources", + "tableTo": "deployment_groups", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_group_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "deployment_group_resources_deployment_group_id_idx_pk": { + "name": "deployment_group_resources_deployment_group_id_idx_pk", + "columns": [ + "deployment_group_id", + "idx" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.deployment_groups": { + "name": "deployment_groups", + "schema": "akash", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "deployment_id": { + "name": "deployment_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gseq": { + "name": "gseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "group_state", + "typeSchema": "akash", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "closed_height": { + "name": "closed_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "deployment_groups_deployment_gseq_idx": { + "name": "deployment_groups_deployment_gseq_idx", + "columns": [ + { + "expression": "deployment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "gseq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "deployment_groups_deployment_id_deployments_id_fk": { + "name": "deployment_groups_deployment_id_deployments_id_fk", + "tableFrom": "deployment_groups", + "tableTo": "deployments", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.deployments": { + "name": "deployments", + "schema": "akash", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "owner_account_id": { + "name": "owner_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "dseq": { + "name": "dseq", + "type": "numeric(20, 0)", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deposit": { + "name": "deposit", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "balance": { + "name": "balance", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "withdrawn_amount": { + "name": "withdrawn_amount", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "block_rate": { + "name": "block_rate", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "last_withdraw_height": { + "name": "last_withdraw_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "last_processed_height": { + "name": "last_processed_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_height": { + "name": "created_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "closed_height": { + "name": "closed_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "closed_at": { + "name": "closed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "close_reason": { + "name": "close_reason", + "type": "deployment_close_reason", + "typeSchema": "akash", + "primaryKey": false, + "notNull": false + }, + "cpu_units": { + "name": "cpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gpu_units": { + "name": "gpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "memory_bytes": { + "name": "memory_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ephemeral_storage_bytes": { + "name": "ephemeral_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "persistent_storage_bytes": { + "name": "persistent_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "deployments_owner_dseq_idx": { + "name": "deployments_owner_dseq_idx", + "columns": [ + { + "expression": "owner_account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dseq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "deployments_owner_created_idx": { + "name": "deployments_owner_created_idx", + "columns": [ + { + "expression": "owner_account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "deployments_open_idx": { + "name": "deployments_open_idx", + "columns": [ + { + "expression": "created_height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"akash\".\"deployments\".\"closed_height\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "deployments_owner_account_id_accounts_id_fk": { + "name": "deployments_owner_account_id_accounts_id_fk", + "tableFrom": "deployments", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "owner_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.indexer_state": { + "name": "indexer_state", + "schema": "", + "columns": { + "stream": { + "name": "stream", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "last_height": { + "name": "last_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.leases": { + "name": "leases", + "schema": "akash", + "columns": { + "deployment_id": { + "name": "deployment_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "deployment_group_id": { + "name": "deployment_group_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gseq": { + "name": "gseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "oseq": { + "name": "oseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "bseq": { + "name": "bseq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "provider_account_id": { + "name": "provider_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "price": { + "name": "price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "balance": { + "name": "balance", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "withdrawn_amount": { + "name": "withdrawn_amount", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "predicted_closed_height": { + "name": "predicted_closed_height", + "type": "numeric(30, 0)", + "primaryKey": false, + "notNull": true + }, + "created_height": { + "name": "created_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "closed_height": { + "name": "closed_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "closed_at": { + "name": "closed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "cpu_units": { + "name": "cpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gpu_units": { + "name": "gpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "memory_bytes": { + "name": "memory_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ephemeral_storage_bytes": { + "name": "ephemeral_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "persistent_storage_bytes": { + "name": "persistent_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "leases_provider_idx": { + "name": "leases_provider_idx", + "columns": [ + { + "expression": "provider_account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "closed_height", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "leases_open_idx": { + "name": "leases_open_idx", + "columns": [ + { + "expression": "deployment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"akash\".\"leases\".\"closed_height\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "leases_deployment_id_deployments_id_fk": { + "name": "leases_deployment_id_deployments_id_fk", + "tableFrom": "leases", + "tableTo": "deployments", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "leases_deployment_group_id_deployment_groups_id_fk": { + "name": "leases_deployment_group_id_deployment_groups_id_fk", + "tableFrom": "leases", + "tableTo": "deployment_groups", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_group_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "leases_provider_account_id_accounts_id_fk": { + "name": "leases_provider_account_id_accounts_id_fk", + "tableFrom": "leases", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "provider_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "leases_deployment_id_gseq_oseq_bseq_provider_account_id_pk": { + "name": "leases_deployment_id_gseq_oseq_bseq_provider_account_id_pk", + "columns": [ + "deployment_id", + "gseq", + "oseq", + "bseq", + "provider_account_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.message_dead_letters": { + "name": "message_dead_letters", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type_id": { + "name": "type_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "raw": { + "name": "raw", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "message_dead_letters_type_id_idx": { + "name": "message_dead_letters_type_id_idx", + "columns": [ + { + "expression": "type_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "message_dead_letters_type_id_message_types_id_fk": { + "name": "message_dead_letters_type_id_message_types_id_fk", + "tableFrom": "message_dead_letters", + "tableTo": "message_types", + "schemaTo": "cosmos", + "columnsFrom": [ + "type_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "message_dead_letters_height_tx_index_index_pk": { + "name": "message_dead_letters_height_tx_index_index_pk", + "columns": [ + "height", + "tx_index", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.message_types": { + "name": "message_types", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "message_types_type_idx": { + "name": "message_types_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.messages": { + "name": "messages", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type_id": { + "name": "type_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "body": { + "name": "body", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "messages_type_id_idx": { + "name": "messages_type_id_idx", + "columns": [ + { + "expression": "type_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "messages_type_id_message_types_id_fk": { + "name": "messages_type_id_message_types_id_fk", + "tableFrom": "messages", + "tableTo": "message_types", + "schemaTo": "cosmos", + "columnsFrom": [ + "type_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messages_height_tx_index_index_pk": { + "name": "messages_height_tx_index_index_pk", + "columns": [ + "height", + "tx_index", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.proposal_deposits": { + "name": "proposal_deposits", + "schema": "cosmos", + "columns": { + "proposal_id": { + "name": "proposal_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "depositor_account_id": { + "name": "depositor_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "amount": { + "name": "amount", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "proposal_deposits_depositor_account_id_accounts_id_fk": { + "name": "proposal_deposits_depositor_account_id_accounts_id_fk", + "tableFrom": "proposal_deposits", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "depositor_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "proposal_deposits_proposal_id_depositor_account_id_height_pk": { + "name": "proposal_deposits_proposal_id_depositor_account_id_height_pk", + "columns": [ + "proposal_id", + "depositor_account_id", + "height" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.proposal_votes": { + "name": "proposal_votes", + "schema": "cosmos", + "columns": { + "proposal_id": { + "name": "proposal_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "voter_account_id": { + "name": "voter_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "options": { + "name": "options", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "proposal_votes_voter_account_id_accounts_id_fk": { + "name": "proposal_votes_voter_account_id_accounts_id_fk", + "tableFrom": "proposal_votes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "voter_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "proposal_votes_proposal_id_voter_account_id_pk": { + "name": "proposal_votes_proposal_id_voter_account_id_pk", + "columns": [ + "proposal_id", + "voter_account_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.proposals": { + "name": "proposals", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "proposer_account_id": { + "name": "proposer_account_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "messages": { + "name": "messages", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "proposal_status", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + }, + "submit_time": { + "name": "submit_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deposit_end_time": { + "name": "deposit_end_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "voting_start_time": { + "name": "voting_start_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "voting_end_time": { + "name": "voting_end_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "total_deposit": { + "name": "total_deposit", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "final_tally_yes": { + "name": "final_tally_yes", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "final_tally_abstain": { + "name": "final_tally_abstain", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "final_tally_no": { + "name": "final_tally_no", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "final_tally_no_with_veto": { + "name": "final_tally_no_with_veto", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "submit_height": { + "name": "submit_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "proposals_proposer_account_id_accounts_id_fk": { + "name": "proposals_proposer_account_id_accounts_id_fk", + "tableFrom": "proposals", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "proposer_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.transactions": { + "name": "transactions", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "code": { + "name": "code", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "gas_used": { + "name": "gas_used", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gas_wanted": { + "name": "gas_wanted", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "fee": { + "name": "fee", + "type": "jsonb", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "transactions_hash_idx": { + "name": "transactions_hash_idx", + "columns": [ + { + "expression": "hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "transactions_height_index_pk": { + "name": "transactions_height_index_pk", + "columns": [ + "height", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.unbonding_delegations": { + "name": "unbonding_delegations", + "schema": "cosmos", + "columns": { + "delegator_account_id": { + "name": "delegator_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "validator_operator_address": { + "name": "validator_operator_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "creation_height": { + "name": "creation_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "completion_time": { + "name": "completion_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "initial_balance": { + "name": "initial_balance", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "balance": { + "name": "balance", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "unbonding_delegations_delegator_account_id_accounts_id_fk": { + "name": "unbonding_delegations_delegator_account_id_accounts_id_fk", + "tableFrom": "unbonding_delegations", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "delegator_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "unbonding_delegations_delegator_account_id_validator_operator_address_creation_height_pk": { + "name": "unbonding_delegations_delegator_account_id_validator_operator_address_creation_height_pk", + "columns": [ + "delegator_account_id", + "validator_operator_address", + "creation_height" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.validators": { + "name": "validators", + "schema": "cosmos", + "columns": { + "operator_address": { + "name": "operator_address", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_address": { + "name": "account_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "hex_address": { + "name": "hex_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "moniker": { + "name": "moniker", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "identity": { + "name": "identity", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "details": { + "name": "details", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "security_contact": { + "name": "security_contact", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commission_rate": { + "name": "commission_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "commission_max_rate": { + "name": "commission_max_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "commission_max_change_rate": { + "name": "commission_max_change_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "min_self_delegation": { + "name": "min_self_delegation", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "jailed": { + "name": "jailed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "status": { + "name": "status", + "type": "validator_status", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": false + }, + "tokens": { + "name": "tokens", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "delegator_shares": { + "name": "delegator_shares", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": false + }, + "unbonding_height": { + "name": "unbonding_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "unbonding_time": { + "name": "unbonding_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "cosmos.account_tx_role": { + "name": "account_tx_role", + "schema": "cosmos", + "values": [ + "signer", + "sender", + "receiver" + ] + }, + "cosmos.balance_change_reason": { + "name": "balance_change_reason", + "schema": "cosmos", + "values": [ + "genesis", + "transfer", + "fee", + "reward", + "commission", + "slash", + "gov", + "ibc", + "escrow", + "bme", + "mint", + "burn", + "staking" + ] + }, + "akash.bid_state": { + "name": "bid_state", + "schema": "akash", + "values": [ + "open", + "active", + "closed" + ] + }, + "akash.deployment_close_reason": { + "name": "deployment_close_reason", + "schema": "akash", + "values": [ + "close_message", + "overdrawn", + "close_event" + ] + }, + "akash.deployment_event_type": { + "name": "deployment_event_type", + "schema": "akash", + "values": [ + "created", + "deposited", + "updated", + "closed", + "group_closed", + "group_paused", + "group_started", + "bid_created", + "bid_closed", + "lease_created", + "lease_closed", + "lease_withdrawn" + ] + }, + "akash.group_state": { + "name": "group_state", + "schema": "akash", + "values": [ + "open", + "paused", + "closed" + ] + }, + "cosmos.proposal_status": { + "name": "proposal_status", + "schema": "cosmos", + "values": [ + "deposit_period", + "voting_period", + "passed", + "rejected", + "failed" + ] + }, + "cosmos.validator_status": { + "name": "validator_status", + "schema": "cosmos", + "values": [ + "unbonded", + "unbonding", + "bonded" + ] + }, + "cosmos.vote_option": { + "name": "vote_option", + "schema": "cosmos", + "values": [ + "yes", + "abstain", + "no", + "no_with_veto" + ] + } + }, + "schemas": { + "akash": "akash", + "cosmos": "cosmos" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/chain-indexer/drizzle/meta/_journal.json b/apps/chain-indexer/drizzle/meta/_journal.json index 7636c3e164..eda4b50e95 100644 --- a/apps/chain-indexer/drizzle/meta/_journal.json +++ b/apps/chain-indexer/drizzle/meta/_journal.json @@ -43,6 +43,13 @@ "when": 1786691068837, "tag": "0005_mysterious_medusa", "breakpoints": true + }, + { + "idx": 6, + "version": "7", + "when": 1786877216729, + "tag": "0006_next_earthquake", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/chain-indexer/package.json b/apps/chain-indexer/package.json index f1f809fd74..dfb1a7fc76 100644 --- a/apps/chain-indexer/package.json +++ b/apps/chain-indexer/package.json @@ -40,6 +40,7 @@ "http-errors": "^2.0.0", "lodash": "^4.17.21", "postgres": "^3.4.4", + "protobufjs": "~6.11.2", "reflect-metadata": "^0.2.2", "tsyringe": "^4.10.0", "undici": "^7.22.0", diff --git a/apps/chain-indexer/src/akash/akash-changes.ts b/apps/chain-indexer/src/akash/akash-changes.ts new file mode 100644 index 0000000000..ad34fe836c --- /dev/null +++ b/apps/chain-indexer/src/akash/akash-changes.ts @@ -0,0 +1,90 @@ +export interface DeploymentKey { + owner: string; + dseq: string; +} + +export interface LeaseKey extends DeploymentKey { + gseq: number; + oseq: number; + bseq: number; + provider: string; +} + +/** A lease/bid's identity within its deployment: the order (gseq, oseq) and the specific bid (bseq, provider), without the owner/dseq. */ +export type LeaseSlot = Pick; + +/** The set of versioned type URLs for one akash message across proto eras, e.g. `/akash.market.v1beta5.MsgCreateBid`. */ +export function akashTypeUrlSet(module: string, name: string, versions: readonly string[]): Set { + return new Set(versions.map(version => `/akash.${module}.${version}.${name}`)); +} + +export interface NormalizedResource { + count: number; + cpuUnits: number; + gpuUnits: number; + gpuVendor: string | null; + gpuModel: string | null; + memoryBytes: number; + ephemeralStorageBytes: number; + persistentStorageBytes: number; + price: string; + priceDenom: string; +} + +export interface NormalizedGroup { + gseq: number; + resources: NormalizedResource[]; +} + +interface ChangeOrigin { + txIndex: number | null; + msgIndex: number | null; +} + +export type AkashChangeBody = + | { kind: "deploymentCreated"; key: DeploymentKey; denom: string; deposit: string; depositor: string | null; groups: NormalizedGroup[] } + | { kind: "deploymentDeposited"; key: DeploymentKey; amount: string; depositor: string | null } + | { kind: "deploymentUpdated"; key: DeploymentKey } + | { kind: "deploymentClosed"; key: DeploymentKey } + | { kind: "groupClosed"; key: DeploymentKey; gseq: number } + | { kind: "groupPaused"; key: DeploymentKey; gseq: number } + | { kind: "groupStarted"; key: DeploymentKey; gseq: number } + | { kind: "bidCreated"; key: LeaseKey; price: string; priceDenom: string } + | { kind: "bidClosed"; key: LeaseKey } + | { kind: "leaseCreated"; key: LeaseKey } + | { kind: "leaseClosed"; key: LeaseKey } + | { kind: "leaseWithdrawn"; key: LeaseKey } + | { kind: "deploymentClosedEvent"; key: DeploymentKey } + | { kind: "leaseClosedEvent"; key: DeploymentKey; gseq: number; oseq: number; bseq: number | null; provider: string }; + +export type AkashChange = AkashChangeBody & ChangeOrigin; + +export type AkashChangeKind = AkashChange["kind"]; + +/** Everything derived from one block, in the exact order the chain applied it (tx order, then message order, then that tx's close events). */ +export interface AkashBlockChanges { + height: number; + datetime: Date; + changes: AkashChange[]; +} + +/** Every address the batch's akash changes reference, for the committer's account interning. */ +export function collectAkashAddresses(blocks: AkashBlockChanges[]): Set { + const addresses = new Set(); + + for (const block of blocks) { + for (const change of block.changes) { + addresses.add(change.key.owner); + if ("provider" in change) { + addresses.add(change.provider); + } else if ("bseq" in change.key) { + addresses.add(change.key.provider); + } + if ("depositor" in change && change.depositor) { + addresses.add(change.depositor); + } + } + } + + return addresses; +} diff --git a/apps/chain-indexer/src/akash/akash-deriver.spec.ts b/apps/chain-indexer/src/akash/akash-deriver.spec.ts new file mode 100644 index 0000000000..b29628df0e --- /dev/null +++ b/apps/chain-indexer/src/akash/akash-deriver.spec.ts @@ -0,0 +1,173 @@ +import { describe, expect, it } from "vitest"; + +import { deriveAkashChanges } from "@src/akash/akash-deriver"; +import type { DecodedBlock, DecodedEvent } from "@src/pipeline/decoded-block"; + +const BLOCK_TIME = new Date("2026-08-13T00:00:00Z"); + +describe("deriveAkashChanges", () => { + it("derives message changes in transaction and message order with their coordinates", () => { + const changes = deriveAkashChanges( + block({ + messages: [ + { + typeUrl: "/akash.deployment.v1beta4.MsgCreateDeployment", + body: { id: { owner: "akash1owner", dseq: "1" }, groups: [], deposit: { amount: { denom: "uakt", amount: "500" } } } + }, + { + typeUrl: "/akash.market.v1beta5.MsgCreateBid", + body: { id: { owner: "akash1owner", dseq: "1", gseq: 1, oseq: 1, bseq: 0, provider: "akash1prov" }, price: { denom: "uakt", amount: "2.5" } } + } + ] + }) + ); + + expect(changes.height).toBe(100); + expect(changes.datetime).toBe(BLOCK_TIME); + expect(changes.changes.map(change => [change.kind, change.txIndex, change.msgIndex])).toEqual([ + ["deploymentCreated", 0, 0], + ["bidCreated", 0, 1] + ]); + }); + + it("skips messages in failed transactions", () => { + const changes = deriveAkashChanges( + block({ + code: 5, + messages: [{ typeUrl: "/akash.deployment.v1beta4.MsgCloseDeployment", body: { id: { owner: "akash1owner", dseq: "1" } } }] + }) + ); + + expect(changes.changes).toEqual([]); + }); + + it("unwraps authz MsgExec through the decoder-provided decoded field, recursively", () => { + const deposit = { + typeUrl: "/akash.escrow.v1.MsgAccountDeposit", + decoded: { signer: "akash1grantee", id: { scope: 1, xid: "akash1owner/7" }, deposit: { amount: { denom: "uakt", amount: "42" } } } + }; + const changes = deriveAkashChanges( + block({ + messages: [ + { + typeUrl: "/cosmos.authz.v1beta1.MsgExec", + body: { grantee: "akash1grantee", msgs: [{ typeUrl: "/cosmos.authz.v1beta1.MsgExec", decoded: { msgs: [deposit] } }] } + } + ] + }) + ); + + expect(changes.changes).toEqual([ + { + kind: "deploymentDeposited", + key: { owner: "akash1owner", dseq: "7" }, + amount: "42", + depositor: "akash1grantee", + txIndex: 0, + msgIndex: 0 + } + ]); + }); + + it("skips exec inner messages the decoder could not decode", () => { + const changes = deriveAkashChanges( + block({ + messages: [ + { + typeUrl: "/cosmos.authz.v1beta1.MsgExec", + body: { msgs: [{ typeUrl: "/akash.escrow.v1.MsgAccountDeposit", value: "AA==", decoded: null }] } + } + ] + }) + ); + + expect(changes.changes).toEqual([]); + }); + + it("derives legacy akash.v1 string close events after the transaction's messages", () => { + const changes = deriveAkashChanges( + block({ + messages: [{ typeUrl: "/akash.deployment.v1beta1.MsgCloseGroup", body: { id: { owner: "akash1owner", dseq: "3", gseq: 1 } } }], + txEvents: [ + event("akash.v1", { action: "lease-closed", owner: "akash1owner", dseq: "3", gseq: "1", oseq: "1", provider: "akash1prov" }), + event("akash.v1", { action: "deployment-closed", owner: "akash1owner", dseq: "3" }) + ] + }) + ); + + expect(changes.changes.map(change => change.kind)).toEqual(["groupClosed", "leaseClosedEvent", "deploymentClosedEvent"]); + expect(changes.changes[1]).toMatchObject({ key: { owner: "akash1owner", dseq: "3" }, gseq: 1, oseq: 1, bseq: null, provider: "akash1prov" }); + }); + + it("derives typed close events from their JSON id attribute", () => { + const changes = deriveAkashChanges( + block({ + txEvents: [ + event( + "akash.market.v1.EventLeaseClosed", + { id: JSON.stringify({ owner: "akash1owner", dseq: "3", gseq: 1, oseq: 1, bseq: 2, provider: "akash1prov" }) }, + 0 + ), + event("akash.deployment.v1.EventDeploymentClosed", { id: JSON.stringify({ owner: "akash1owner", dseq: "3" }) }) + ] + }) + ); + + expect(changes.changes).toEqual([ + { kind: "leaseClosedEvent", key: { owner: "akash1owner", dseq: "3" }, gseq: 1, oseq: 1, bseq: 2, provider: "akash1prov", txIndex: 0, msgIndex: 0 }, + { kind: "deploymentClosedEvent", key: { owner: "akash1owner", dseq: "3" }, txIndex: 0, msgIndex: null } + ]); + }); + + it("ignores malformed close events and unrelated event types", () => { + const changes = deriveAkashChanges( + block({ + txEvents: [ + event("akash.v1", { action: "deployment-closed" }), + event("akash.deployment.v1.EventDeploymentClosed", { id: "not-json" }), + event("transfer", { amount: "1uakt" }) + ] + }) + ); + + expect(changes.changes).toEqual([]); + }); + + function block(input: { + height?: number; + code?: number; + messages?: { typeUrl: string; body: unknown }[]; + txEvents?: DecodedEvent[]; + blockEvents?: DecodedEvent[]; + }): DecodedBlock { + const messages = input.messages ?? []; + return { + height: input.height ?? 100, + datetime: BLOCK_TIME, + hash: Buffer.alloc(0), + parentHash: null, + proposerAddress: "P", + transactions: + messages.length > 0 || input.txEvents + ? [ + { + index: 0, + hash: Buffer.alloc(0), + code: input.code ?? 0, + gasUsed: 0, + gasWanted: 0, + fee: [], + messages: messages.map((message, index) => ({ index, typeUrl: message.typeUrl, body: message.body })), + events: input.txEvents ?? [], + signerAddresses: [] + } + ] + : [], + blockEvents: input.blockEvents ?? [] + }; + } + + function event(type: string, attributes: Record, msgIndex?: number): DecodedEvent { + return msgIndex === undefined ? { type, attributes } : { type, attributes, msgIndex }; + } +}); diff --git a/apps/chain-indexer/src/akash/akash-deriver.ts b/apps/chain-indexer/src/akash/akash-deriver.ts new file mode 100644 index 0000000000..6f600ce495 --- /dev/null +++ b/apps/chain-indexer/src/akash/akash-deriver.ts @@ -0,0 +1,142 @@ +import type { AkashBlockChanges, AkashChange, AkashChangeBody } from "@src/akash/akash-changes"; +import { asInteger, asRecord, asString } from "@src/akash/json"; +import { normalizeDeploymentMessage } from "@src/akash/normalize-deployment"; +import { normalizeMarketMessage } from "@src/akash/normalize-market"; +import { asUint64String } from "@src/akash/uint64"; +import type { DecodedBlock, DecodedEvent } from "@src/pipeline/decoded-block"; +import { MAX_EXEC_DEPTH, MSG_EXEC_TYPE_URL } from "@src/pipeline/msg-exec"; + +const LEGACY_EVENT_TYPE = "akash.v1"; +const DEPLOYMENT_CLOSED_EVENT_TYPE = "akash.deployment.v1.EventDeploymentClosed"; +const LEASE_CLOSED_EVENT_TYPE = "akash.market.v1.EventLeaseClosed"; + +/** + * Extracts the deployment and market lifecycle from a block's messages and close events, in the exact + * order the chain applied it: per transaction, messages first (authz MsgExec unwrapped through the + * decoder-provided `decoded` field), then that transaction's close events, which catch deployment and + * lease closes happening as side effects (group close, authz revoke, overdraw on withdraw). Messages + * in failed transactions are skipped, since cosmos rolls back their state changes and no close event + * is emitted for them. + */ +export function deriveAkashChanges(block: DecodedBlock): AkashBlockChanges { + const changes: AkashChange[] = []; + + for (const tx of block.transactions) { + if (tx.code !== 0) { + continue; + } + for (const message of tx.messages) { + addMessage(changes, message.typeUrl, message.body, tx.index, message.index, 0); + } + addCloseEvents(changes, tx.events, tx.index); + } + + addCloseEvents(changes, block.blockEvents, null); + + return { height: block.height, datetime: block.datetime, changes }; +} + +function addMessage(changes: AkashChange[], typeUrl: string, body: unknown, txIndex: number, msgIndex: number, depth: number): void { + if (typeUrl === MSG_EXEC_TYPE_URL && depth < MAX_EXEC_DEPTH) { + const msgs = asRecord(body)?.msgs; + if (Array.isArray(msgs)) { + for (const inner of msgs) { + const innerRecord = asRecord(inner); + const innerTypeUrl = asString(innerRecord?.typeUrl); + if (innerTypeUrl && innerRecord?.decoded) { + addMessage(changes, innerTypeUrl, innerRecord.decoded, txIndex, msgIndex, depth + 1); + } + } + } + return; + } + + const record = asRecord(body); + if (!record) { + return; + } + + const normalized = normalizeDeploymentMessage(typeUrl, record) ?? normalizeMarketMessage(typeUrl, record); + + if (normalized) { + changes.push({ ...normalized, txIndex, msgIndex }); + } +} + +function addCloseEvents(changes: AkashChange[], events: DecodedEvent[], txIndex: number | null): void { + for (const event of events) { + const change = closeEventChange(event); + if (change) { + changes.push({ ...change, txIndex, msgIndex: event.msgIndex ?? null }); + } + } +} + +function closeEventChange(event: DecodedEvent): AkashChangeBody | null { + if (event.type === LEGACY_EVENT_TYPE) { + if (event.attributes.action === "deployment-closed") { + return legacyDeploymentClosed(event.attributes); + } + if (event.attributes.action === "lease-closed") { + return legacyLeaseClosed(event.attributes); + } + return null; + } + if (event.type === DEPLOYMENT_CLOSED_EVENT_TYPE) { + return typedDeploymentClosed(event.attributes); + } + if (event.type === LEASE_CLOSED_EVENT_TYPE) { + return typedLeaseClosed(event.attributes); + } + return null; +} + +function legacyDeploymentClosed(attributes: Record): AkashChangeBody | null { + const owner = asString(attributes.owner); + const dseq = asUint64String(attributes.dseq); + return owner && dseq ? { kind: "deploymentClosedEvent", key: { owner, dseq } } : null; +} + +function legacyLeaseClosed(attributes: Record): AkashChangeBody | null { + const owner = asString(attributes.owner); + const dseq = asUint64String(attributes.dseq); + const gseq = asInteger(attributes.gseq); + const oseq = asInteger(attributes.oseq); + const provider = asString(attributes.provider); + if (!owner || !dseq || gseq === null || oseq === null || !provider) { + return null; + } + return { kind: "leaseClosedEvent", key: { owner, dseq }, gseq, oseq, bseq: null, provider }; +} + +function typedDeploymentClosed(attributes: Record): AkashChangeBody | null { + const id = parseIdAttribute(attributes.id); + const owner = asString(id?.owner); + const dseq = asUint64String(id?.dseq); + return owner && dseq ? { kind: "deploymentClosedEvent", key: { owner, dseq } } : null; +} + +function typedLeaseClosed(attributes: Record): AkashChangeBody | null { + const id = parseIdAttribute(attributes.id); + const owner = asString(id?.owner); + const dseq = asUint64String(id?.dseq); + const gseq = asInteger(id?.gseq); + const oseq = asInteger(id?.oseq); + const provider = asString(id?.provider); + if (!owner || !dseq || gseq === null || oseq === null || !provider) { + return null; + } + return { kind: "leaseClosedEvent", key: { owner, dseq }, gseq, oseq, bseq: asInteger(id?.bseq), provider }; +} + +/** The typed events carry their id as a JSON string attribute. */ +function parseIdAttribute(raw: string | undefined): Record | null { + if (!raw) { + return null; + } + try { + return asRecord(JSON.parse(raw)); + } catch { + return null; + } +} diff --git a/apps/chain-indexer/src/akash/akash-writer.service.spec.ts b/apps/chain-indexer/src/akash/akash-writer.service.spec.ts new file mode 100644 index 0000000000..2f332f3321 --- /dev/null +++ b/apps/chain-indexer/src/akash/akash-writer.service.spec.ts @@ -0,0 +1,301 @@ +import type { SQL } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { describe, expect, it } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import type { AkashBlockChanges, AkashChangeBody } from "@src/akash/akash-changes"; +import { AkashWriter } from "@src/akash/akash-writer.service"; +import { Bids, DeploymentEvents, DeploymentGroupResources, DeploymentGroups, Deployments, Leases } from "@src/db/schema"; +import type { ChainTransaction } from "@src/providers/db.provider"; +import type { LoggerService } from "@src/providers/logging.provider"; + +const OWNER = "akash1owner"; +const PROVIDER = "akash1prov"; +const KEY = { owner: OWNER, dseq: "42" }; +const LEASE_KEY = { ...KEY, gseq: 1, oseq: 1, bseq: 0, provider: PROVIDER }; +const BLOCK_TIME = new Date("2026-08-13T00:00:00Z"); +const ACCOUNT_IDS = new Map([ + [OWNER, 7], + [PROVIDER, 8] +]); + +describe(AkashWriter.name, () => { + it("does nothing for blocks without akash changes", async () => { + const { writer, tx, inserts, selects } = setup(); + + await writer.write(tx, [block(100, [])], ACCOUNT_IDS); + + expect(inserts).toEqual([]); + expect(selects).toEqual([]); + }); + + it("persists a full lifecycle batch as one consistent set of rows", async () => { + const { writer, tx, inserts, upserts } = setup(); + + await writer.write( + tx, + [block(100, [create(), bidCreated("10")]), block(110, [{ kind: "leaseCreated", key: LEASE_KEY }]), block(200, [{ kind: "deploymentClosed", key: KEY }])], + ACCOUNT_IDS + ); + + const [deploymentRow] = rowsFor(inserts, Deployments); + expect(deploymentRow).toMatchObject({ + ownerAccountId: 7, + dseq: "42", + denom: "uakt", + deposit: "5000000", + balance: "4999100", + withdrawnAmount: "900", + blockRate: "0", + lastWithdrawHeight: 200, + lastProcessedHeight: 200, + createdHeight: 100, + closedHeight: 200, + closeReason: "close_message", + cpuUnits: 2000 + }); + + expect(rowsFor(inserts, DeploymentGroups)).toEqual([{ deploymentId: 1, gseq: 1, state: "open", closedHeight: null }]); + expect(rowsFor(inserts, DeploymentGroupResources)).toEqual([ + expect.objectContaining({ deploymentGroupId: 2, idx: 0, count: 2, cpuUnits: 1000, price: "1" }) + ]); + expect(rowsFor(inserts, Bids)).toEqual([ + expect.objectContaining({ deploymentId: 1, providerAccountId: 8, price: "10", state: "closed", closedHeight: 200 }) + ]); + expect(rowsFor(inserts, Leases)).toEqual([ + expect.objectContaining({ + deploymentId: 1, + deploymentGroupId: 2, + providerAccountId: 8, + price: "10", + withdrawnAmount: "900", + createdHeight: 110, + closedHeight: 200, + cpuUnits: 2000 + }) + ]); + expect(rowsFor(inserts, DeploymentEvents).map(row => [row.type, row.height, row.ordinal])).toEqual([ + ["created", 100, 0], + ["bid_created", 100, 1], + ["lease_created", 110, 0], + ["closed", 200, 0] + ]); + + const deploymentUpsert = upserts.find(upsert => upsert.table === Deployments); + expect(whereSql(deploymentUpsert?.config.setWhere as SQL)).toContain('excluded.last_processed_height >= "akash"."deployments"."last_processed_height"'); + }); + + it("skips flushing entirely when every block is at or below the stored watermark", async () => { + const { writer, tx, inserts } = setup({ + deployments: [deploymentRow({ lastProcessedHeight: 500 })] + }); + + await writer.write(tx, [block(400, [{ kind: "deploymentDeposited", key: KEY, amount: "10", depositor: null }])], ACCOUNT_IDS); + + expect(inserts).toEqual([]); + }); + + it("logs orphan references without aborting the batch", async () => { + const { writer, tx, logger, inserts } = setup(); + + await writer.write(tx, [block(100, [{ kind: "leaseWithdrawn", key: LEASE_KEY }])], ACCOUNT_IDS); + + expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ event: "AKASH_ORPHAN_REFERENCE", count: 1 })); + expect(inserts).toEqual([]); + }); + + it("applies new blocks on top of loaded state", async () => { + const { writer, tx, inserts } = setup({ + deployments: [deploymentRow({ lastProcessedHeight: 110, lastWithdrawHeight: 110, balance: "1000.000000000000000000" })], + leases: [leaseRow()] + }); + + await writer.write(tx, [block(150, [{ kind: "leaseWithdrawn", key: LEASE_KEY }])], ACCOUNT_IDS); + + const [row] = rowsFor(inserts, Deployments); + expect(row).toMatchObject({ balance: "600", withdrawnAmount: "400", lastWithdrawHeight: 150, lastProcessedHeight: 150 }); + const [lease] = rowsFor(inserts, Leases); + expect(lease).toMatchObject({ withdrawnAmount: "400" }); + }); + + it("re-selects the deployment id when a guarded upsert returns no row", async () => { + const { writer, tx, inserts } = setup({ returningEmpty: true }); + + await writer.write(tx, [block(100, [create(), bidCreated("10")]), block(110, [{ kind: "leaseCreated", key: LEASE_KEY }])], ACCOUNT_IDS); + + const [lease] = rowsFor(inserts, Leases); + expect(lease).toMatchObject({ deploymentId: 1 }); + }); + + function setup(input?: { + deployments?: Record[]; + groups?: Record[]; + bids?: Record[]; + leases?: Record[]; + returningEmpty?: boolean; + }) { + const inserts: { table: unknown; rows: Record[] }[] = []; + const upserts: { table: unknown; config: Record }[] = []; + const selects: unknown[] = []; + let nextId = 1; + let deploymentSelects = 0; + + const deployments = input?.deployments ?? []; + const rowsByTable = new Map[]>([ + [Deployments, deployments], + [DeploymentGroups, input?.groups ?? (deployments.length > 0 ? [{ id: 2, deploymentId: 1, gseq: 1, state: "open", closedHeight: null }] : [])], + [Bids, input?.bids ?? []], + [Leases, input?.leases ?? []] + ]); + + const selectChain = (table: unknown) => { + if (table === Deployments) { + deploymentSelects++; + if (input?.returningEmpty && deploymentSelects === 2) { + rowsByTable.set( + Deployments, + rowsFor(inserts, Deployments).map((row, index) => ({ id: index + 1, ...row })) + ); + } + } + const rows = rowsByTable.get(table) ?? providerAccountRows(table); + const chain = { + where: () => chain, + orderBy: () => chain, + innerJoin: () => chain, + for: () => chain, + then: (resolve: (rows: unknown[]) => unknown, reject?: (error: unknown) => unknown) => Promise.resolve(rows).then(resolve, reject) + }; + return chain; + }; + + const providerAccountRows = (table: unknown) => { + void table; + return [ + { id: 7, address: OWNER }, + { id: 8, address: PROVIDER } + ]; + }; + + const tx = { + insert: (table: unknown) => ({ + values: (rows: Record | Record[]) => { + const rowArray = Array.isArray(rows) ? rows : [rows]; + inserts.push({ table, rows: rowArray }); + const returning = () => Promise.resolve(input?.returningEmpty && table === Deployments ? [] : rowArray.map(row => ({ id: nextId++, ...row }))); + return Object.assign(Promise.resolve(), { + returning, + onConflictDoNothing: () => Object.assign(Promise.resolve(), { returning }), + onConflictDoUpdate: (config: Record) => { + upserts.push({ table, config }); + return Object.assign(Promise.resolve(), { returning }); + } + }); + } + }), + select: (fields?: unknown) => { + selects.push(fields); + return { from: (table: unknown) => selectChain(table) }; + } + }; + + const logger = mock(); + return { writer: new AkashWriter(logger), tx: tx as unknown as ChainTransaction, inserts, upserts, selects, logger }; + } + + function deploymentRow(overrides: Record) { + return { + id: 1, + ownerAccountId: 7, + dseq: "42", + denom: "uakt", + deposit: "1000", + balance: "1000.000000000000000000", + withdrawnAmount: "0.000000000000000000", + blockRate: "10.000000000000000000", + lastWithdrawHeight: null, + lastProcessedHeight: 100, + createdHeight: 100, + createdAt: BLOCK_TIME, + closedHeight: null, + closedAt: null, + closeReason: null, + cpuUnits: 2000, + gpuUnits: 0, + memoryBytes: 0, + ephemeralStorageBytes: 0, + persistentStorageBytes: 0, + ...overrides + }; + } + + function leaseRow() { + return { + deploymentId: 1, + deploymentGroupId: 2, + gseq: 1, + oseq: 1, + bseq: 0, + providerAccountId: 8, + price: "10.000000000000000000", + denom: "uakt", + balance: "0.000000000000000000", + withdrawnAmount: "0.000000000000000000", + predictedClosedHeight: "210", + createdHeight: 110, + createdAt: BLOCK_TIME, + closedHeight: null, + closedAt: null, + cpuUnits: 2000, + gpuUnits: 0, + memoryBytes: 0, + ephemeralStorageBytes: 0, + persistentStorageBytes: 0 + }; + } + + function block(height: number, bodies: AkashChangeBody[]): AkashBlockChanges { + return { height, datetime: BLOCK_TIME, changes: bodies.map((body, index) => ({ ...body, txIndex: 0, msgIndex: index })) }; + } + + function create(): AkashChangeBody { + return { + kind: "deploymentCreated", + key: KEY, + denom: "uakt", + deposit: "5000000", + depositor: null, + groups: [ + { + gseq: 1, + resources: [ + { + count: 2, + cpuUnits: 1000, + gpuUnits: 0, + gpuVendor: null, + gpuModel: null, + memoryBytes: 0, + ephemeralStorageBytes: 0, + persistentStorageBytes: 0, + price: "1", + priceDenom: "uakt" + } + ] + } + ] + }; + } + + function bidCreated(price: string): AkashChangeBody { + return { kind: "bidCreated", key: LEASE_KEY, price, priceDenom: "uakt" }; + } + + function rowsFor(inserts: { table: unknown; rows: Record[] }[], table: unknown): Record[] { + return inserts.filter(insert => insert.table === table).flatMap(insert => insert.rows); + } + + function whereSql(where: SQL): string { + return new PgDialect().sqlToQuery(where).sql; + } +}); diff --git a/apps/chain-indexer/src/akash/akash-writer.service.ts b/apps/chain-indexer/src/akash/akash-writer.service.ts new file mode 100644 index 0000000000..1471145dd1 --- /dev/null +++ b/apps/chain-indexer/src/akash/akash-writer.service.ts @@ -0,0 +1,525 @@ +import { and, eq, inArray, or, sql } from "drizzle-orm"; +import groupBy from "lodash/groupBy"; +import { inject, singleton } from "tsyringe"; + +import type { AkashBlockChanges, DeploymentKey, NormalizedResource } from "@src/akash/akash-changes"; +import { decFromString, decToString } from "@src/akash/dec"; +import type { BidStateValue, DeploymentAggState, GroupStateValue, ReducerWarning } from "@src/akash/deployment-reducer"; +import { applyBlockChanges, stateKey } from "@src/akash/deployment-reducer"; +import { sumLeaseRate } from "@src/akash/settlement"; +import { insertChunked } from "@src/db/insert-chunked"; +import { Accounts, Bids, DeploymentEvents, DeploymentGroupResources, DeploymentGroups, Deployments, Leases } from "@src/db/schema"; +import { sqlExcluded } from "@src/db/sql-excluded"; +import type { ChainTransaction } from "@src/providers/db.provider"; +import { LoggerService } from "@src/providers/logging.provider"; + +interface KeyedDeployment { + key: DeploymentKey; + ownerAccountId: number; +} + +/** + * Persists the deployment and market lifecycle inside the block transaction. The touched deployment + * rows are locked `FOR UPDATE` in a deterministic order (so overlapping writers serialize instead of + * deadlocking), folded in memory through the reducer in strict block order, and flushed as guarded + * upserts. The per-deployment `last_processed_height` watermark makes a duplicate commit (replay, + * overlapping pod) a no-op; like the balance ledger, this requires a deployment's messages to be + * indexed in height order from its creation (backfill from genesis before live sync). + */ +@singleton() +export class AkashWriter { + readonly #logger: LoggerService; + + constructor(@inject(LoggerService) logger: LoggerService) { + this.#logger = logger; + this.#logger.setContext("AKASH_WRITER"); + } + + async write(tx: ChainTransaction, blocks: AkashBlockChanges[], accountIds: Map): Promise { + const withChanges = blocks.filter(block => block.changes.length > 0); + if (withChanges.length === 0) { + return; + } + + const keyed = this.#collectKeys(withChanges, accountIds); + const { states, deploymentIds, groupIds, loadedAddressIds } = await this.#loadStates(tx, keyed); + + const warnings = withChanges.flatMap(block => applyBlockChanges(states, block)); + this.#logWarnings(warnings); + + const touched = [...states.values()].filter(state => state.touched); + if (touched.length === 0) { + return; + } + + /** Providers of bids and leases loaded from prior batches aren't in this batch's interned map, so their ids come from the loaded rows. */ + const addressIds = new Map([...loadedAddressIds, ...accountIds]); + + await this.#flushDeployments(tx, touched, addressIds, deploymentIds); + await this.#flushGroups(tx, touched, deploymentIds, groupIds); + await this.#flushGroupResources(tx, touched, deploymentIds, groupIds); + await this.#flushBids(tx, touched, addressIds, deploymentIds); + await this.#flushLeases(tx, touched, addressIds, deploymentIds, groupIds); + await this.#flushEvents(tx, touched, deploymentIds); + } + + /** Deterministic (ownerAccountId, dseq) order for both the row locks and the flush statements, so concurrent writers cannot deadlock. */ + #collectKeys(blocks: AkashBlockChanges[], accountIds: Map): KeyedDeployment[] { + const byKey = new Map(); + for (const block of blocks) { + for (const change of block.changes) { + byKey.set(stateKey(change.key), change.key); + } + } + + return [...byKey.values()] + .map(key => ({ key, ownerAccountId: this.#requireId(accountIds, key.owner) })) + .sort((a, b) => a.ownerAccountId - b.ownerAccountId || compareDseq(a.key.dseq, b.key.dseq)); + } + + async #loadStates( + tx: ChainTransaction, + keyed: KeyedDeployment[] + ): Promise<{ + states: Map; + deploymentIds: Map; + groupIds: Map; + loadedAddressIds: Map; + }> { + const states = new Map(); + const deploymentIds = new Map(); + const groupIds = new Map(); + const loadedAddressIds = new Map(); + + const deploymentRows = await this.#selectDeploymentsForUpdate(tx, keyed); + if (deploymentRows.length === 0) { + return { states, deploymentIds, groupIds, loadedAddressIds }; + } + + const keyByOwnerDseq = new Map(keyed.map(entry => [ownerDseqKey(entry.ownerAccountId, entry.key.dseq), entry.key])); + const ids = deploymentRows.map(row => row.id); + const [groupRows, resourceRows, bidRows, leaseRows] = await Promise.all([ + tx.select().from(DeploymentGroups).where(inArray(DeploymentGroups.deploymentId, ids)), + tx + .select({ resource: DeploymentGroupResources, deploymentId: DeploymentGroups.deploymentId, gseq: DeploymentGroups.gseq }) + .from(DeploymentGroupResources) + .innerJoin(DeploymentGroups, eq(DeploymentGroupResources.deploymentGroupId, DeploymentGroups.id)) + .where(inArray(DeploymentGroups.deploymentId, ids)), + tx.select().from(Bids).where(inArray(Bids.deploymentId, ids)), + tx.select().from(Leases).where(inArray(Leases.deploymentId, ids)) + ]); + + const providerAddressById = await this.#providerAddresses(tx, [ + ...bidRows.map(row => row.providerAccountId), + ...leaseRows.map(row => row.providerAccountId) + ]); + for (const [id, address] of providerAddressById) { + loadedAddressIds.set(address, id); + } + + const groupsByDeployment = groupBy(groupRows, row => row.deploymentId); + const resourcesByGroup = groupBy(resourceRows, entry => `${entry.deploymentId}/${entry.gseq}`); + const bidsByDeployment = groupBy(bidRows, row => row.deploymentId); + const leasesByDeployment = groupBy(leaseRows, row => row.deploymentId); + + for (const row of deploymentRows) { + const key = keyByOwnerDseq.get(ownerDseqKey(row.ownerAccountId, row.dseq)); + if (!key) { + continue; + } + deploymentIds.set(stateKey(key), row.id); + + const groups = groupsByDeployment[row.id] ?? []; + for (const group of groups) { + groupIds.set(`${row.id}/${group.gseq}`, group.id); + } + + states.set(stateKey(key), { + key, + denom: row.denom, + deposit: BigInt(row.deposit), + balance: decFromString(row.balance), + withdrawn: decFromString(row.withdrawnAmount), + lastWithdrawHeight: row.lastWithdrawHeight, + lastProcessedHeight: row.lastProcessedHeight, + createdHeight: row.createdHeight, + createdAt: row.createdAt, + closedHeight: row.closedHeight, + closedAt: row.closedAt, + closeReason: row.closeReason, + cpuUnits: row.cpuUnits, + gpuUnits: row.gpuUnits, + memoryBytes: row.memoryBytes, + ephemeralStorageBytes: row.ephemeralStorageBytes, + persistentStorageBytes: row.persistentStorageBytes, + groups: groups.map(group => ({ + gseq: group.gseq, + state: group.state as GroupStateValue, + closedHeight: group.closedHeight, + resources: (resourcesByGroup[`${row.id}/${group.gseq}`] ?? []) + .sort((a, b) => a.resource.idx - b.resource.idx) + .map(entry => toNormalizedResource(entry.resource)) + })), + bids: (bidsByDeployment[row.id] ?? []).map(bid => ({ + gseq: bid.gseq, + oseq: bid.oseq, + bseq: bid.bseq, + provider: this.#requireAddress(providerAddressById, bid.providerAccountId), + price: decFromString(bid.price), + denom: bid.denom, + state: bid.state as BidStateValue, + createdHeight: bid.createdHeight, + closedHeight: bid.closedHeight + })), + leases: (leasesByDeployment[row.id] ?? []).map(lease => ({ + gseq: lease.gseq, + oseq: lease.oseq, + bseq: lease.bseq, + provider: this.#requireAddress(providerAddressById, lease.providerAccountId), + price: decFromString(lease.price), + denom: lease.denom, + balance: decFromString(lease.balance), + withdrawn: decFromString(lease.withdrawnAmount), + predictedClosedHeight: BigInt(lease.predictedClosedHeight), + createdHeight: lease.createdHeight, + createdAt: lease.createdAt, + closedHeight: lease.closedHeight, + closedAt: lease.closedAt, + cpuUnits: lease.cpuUnits, + gpuUnits: lease.gpuUnits, + memoryBytes: lease.memoryBytes, + ephemeralStorageBytes: lease.ephemeralStorageBytes, + persistentStorageBytes: lease.persistentStorageBytes + })), + events: [], + isNew: false, + touched: false + }); + } + + return { states, deploymentIds, groupIds, loadedAddressIds }; + } + + async #selectDeploymentsForUpdate(tx: ChainTransaction, keyed: KeyedDeployment[]) { + const filters = keyed.map(entry => and(eq(Deployments.ownerAccountId, entry.ownerAccountId), eq(Deployments.dseq, entry.key.dseq))); + return tx + .select() + .from(Deployments) + .where(or(...filters)) + .orderBy(Deployments.ownerAccountId, Deployments.dseq) + .for("update"); + } + + /** Bid and lease provider addresses are only stored as account ids; the reducer keys leases by address, so resolve them back. */ + async #providerAddresses(tx: ChainTransaction, providerAccountIds: number[]): Promise> { + const unique = [...new Set(providerAccountIds)]; + if (unique.length === 0) { + return new Map(); + } + const rows = await tx.select({ id: Accounts.id, address: Accounts.address }).from(Accounts).where(inArray(Accounts.id, unique)); + return new Map(rows.map(row => [row.id, row.address])); + } + + async #flushDeployments( + tx: ChainTransaction, + touched: DeploymentAggState[], + accountIds: Map, + deploymentIds: Map + ): Promise { + const rows = touched.map(state => ({ + ownerAccountId: this.#requireId(accountIds, state.key.owner), + dseq: state.key.dseq, + denom: state.denom, + deposit: state.deposit.toString(), + balance: decToString(state.balance), + withdrawnAmount: decToString(state.withdrawn), + blockRate: decToString(sumLeaseRate(state.leases.filter(lease => lease.closedHeight === null))), + lastWithdrawHeight: state.lastWithdrawHeight, + lastProcessedHeight: state.lastProcessedHeight, + createdHeight: state.createdHeight, + createdAt: state.createdAt, + closedHeight: state.closedHeight, + closedAt: state.closedAt, + closeReason: state.closeReason, + cpuUnits: state.cpuUnits, + gpuUnits: state.gpuUnits, + memoryBytes: state.memoryBytes, + ephemeralStorageBytes: state.ephemeralStorageBytes, + persistentStorageBytes: state.persistentStorageBytes + })); + + const inserted = await tx + .insert(Deployments) + .values(rows) + .onConflictDoUpdate({ + target: [Deployments.ownerAccountId, Deployments.dseq], + set: { + denom: sqlExcluded("denom"), + deposit: sqlExcluded("deposit"), + balance: sqlExcluded("balance"), + withdrawnAmount: sqlExcluded("withdrawn_amount"), + blockRate: sqlExcluded("block_rate"), + lastWithdrawHeight: sqlExcluded("last_withdraw_height"), + lastProcessedHeight: sqlExcluded("last_processed_height"), + closedHeight: sqlExcluded("closed_height"), + closedAt: sqlExcluded("closed_at"), + closeReason: sqlExcluded("close_reason") + }, + setWhere: sql`excluded.last_processed_height >= ${Deployments.lastProcessedHeight}` + }) + .returning({ id: Deployments.id, ownerAccountId: Deployments.ownerAccountId, dseq: Deployments.dseq }); + + const idByOwnerDseq = new Map(inserted.map(row => [ownerDseqKey(row.ownerAccountId, row.dseq), row.id])); + for (const state of touched) { + const id = idByOwnerDseq.get(ownerDseqKey(this.#requireId(accountIds, state.key.owner), state.key.dseq)); + if (id !== undefined) { + deploymentIds.set(stateKey(state.key), id); + } + } + + const missing = touched.filter(state => !deploymentIds.has(stateKey(state.key))); + if (missing.length > 0) { + const rowsForMissing = await this.#selectDeploymentsForUpdate( + tx, + missing.map(state => ({ key: state.key, ownerAccountId: this.#requireId(accountIds, state.key.owner) })) + ); + const keyByOwnerDseq = new Map(missing.map(state => [ownerDseqKey(this.#requireId(accountIds, state.key.owner), state.key.dseq), state.key])); + for (const row of rowsForMissing) { + const key = keyByOwnerDseq.get(ownerDseqKey(row.ownerAccountId, row.dseq)); + if (key) { + deploymentIds.set(stateKey(key), row.id); + } + } + } + } + + async #flushGroups(tx: ChainTransaction, touched: DeploymentAggState[], deploymentIds: Map, groupIds: Map): Promise { + const rows = touched.flatMap(state => { + const deploymentId = this.#requireDeploymentId(deploymentIds, state); + return state.groups.map(group => ({ deploymentId, gseq: group.gseq, state: group.state, closedHeight: group.closedHeight })); + }); + if (rows.length === 0) { + return; + } + + const affected = await tx + .insert(DeploymentGroups) + .values(rows) + .onConflictDoUpdate({ + target: [DeploymentGroups.deploymentId, DeploymentGroups.gseq], + set: { state: sqlExcluded("state"), closedHeight: sqlExcluded("closed_height") } + }) + .returning({ id: DeploymentGroups.id, deploymentId: DeploymentGroups.deploymentId, gseq: DeploymentGroups.gseq }); + + for (const row of affected) { + groupIds.set(`${row.deploymentId}/${row.gseq}`, row.id); + } + } + + async #flushGroupResources( + tx: ChainTransaction, + touched: DeploymentAggState[], + deploymentIds: Map, + groupIds: Map + ): Promise { + const rows = touched + .filter(state => state.isNew) + .flatMap(state => + state.groups.flatMap(group => + group.resources.map((resource, idx) => ({ + deploymentGroupId: this.#requireGroupId(groupIds, this.#requireDeploymentId(deploymentIds, state), group.gseq), + idx, + count: resource.count, + cpuUnits: resource.cpuUnits, + gpuUnits: resource.gpuUnits, + gpuVendor: resource.gpuVendor, + gpuModel: resource.gpuModel, + memoryBytes: resource.memoryBytes, + ephemeralStorageBytes: resource.ephemeralStorageBytes, + persistentStorageBytes: resource.persistentStorageBytes, + price: resource.price, + priceDenom: resource.priceDenom + })) + ) + ); + await insertChunked(tx, DeploymentGroupResources, rows); + } + + async #flushBids(tx: ChainTransaction, touched: DeploymentAggState[], accountIds: Map, deploymentIds: Map): Promise { + const rows = touched.flatMap(state => { + const deploymentId = this.#requireDeploymentId(deploymentIds, state); + return state.bids.map(bid => ({ + deploymentId, + gseq: bid.gseq, + oseq: bid.oseq, + bseq: bid.bseq, + providerAccountId: this.#requireId(accountIds, bid.provider), + price: decToString(bid.price), + denom: bid.denom, + state: bid.state, + createdHeight: bid.createdHeight, + closedHeight: bid.closedHeight + })); + }); + if (rows.length === 0) { + return; + } + + await tx + .insert(Bids) + .values(rows) + .onConflictDoUpdate({ + target: [Bids.deploymentId, Bids.gseq, Bids.oseq, Bids.bseq, Bids.providerAccountId], + set: { + price: sqlExcluded("price"), + denom: sqlExcluded("denom"), + state: sqlExcluded("state"), + createdHeight: sqlExcluded("created_height"), + closedHeight: sqlExcluded("closed_height") + } + }); + } + + async #flushLeases( + tx: ChainTransaction, + touched: DeploymentAggState[], + accountIds: Map, + deploymentIds: Map, + groupIds: Map + ): Promise { + const rows = touched.flatMap(state => { + const deploymentId = this.#requireDeploymentId(deploymentIds, state); + return state.leases.map(lease => ({ + deploymentId, + deploymentGroupId: this.#requireGroupId(groupIds, deploymentId, lease.gseq), + gseq: lease.gseq, + oseq: lease.oseq, + bseq: lease.bseq, + providerAccountId: this.#requireId(accountIds, lease.provider), + price: decToString(lease.price), + denom: lease.denom, + balance: decToString(lease.balance), + withdrawnAmount: decToString(lease.withdrawn), + predictedClosedHeight: lease.predictedClosedHeight.toString(), + createdHeight: lease.createdHeight, + createdAt: lease.createdAt, + closedHeight: lease.closedHeight, + closedAt: lease.closedAt, + cpuUnits: lease.cpuUnits, + gpuUnits: lease.gpuUnits, + memoryBytes: lease.memoryBytes, + ephemeralStorageBytes: lease.ephemeralStorageBytes, + persistentStorageBytes: lease.persistentStorageBytes + })); + }); + if (rows.length === 0) { + return; + } + + await tx + .insert(Leases) + .values(rows) + .onConflictDoUpdate({ + target: [Leases.deploymentId, Leases.gseq, Leases.oseq, Leases.bseq, Leases.providerAccountId], + set: { + balance: sqlExcluded("balance"), + withdrawnAmount: sqlExcluded("withdrawn_amount"), + predictedClosedHeight: sqlExcluded("predicted_closed_height"), + closedHeight: sqlExcluded("closed_height"), + closedAt: sqlExcluded("closed_at") + } + }); + } + + async #flushEvents(tx: ChainTransaction, touched: DeploymentAggState[], deploymentIds: Map): Promise { + const rows = touched.flatMap(state => { + const deploymentId = this.#requireDeploymentId(deploymentIds, state); + return state.events.map(event => ({ + deploymentId, + height: event.height, + ordinal: event.ordinal, + txIndex: event.txIndex, + msgIndex: event.msgIndex, + type: event.type, + details: event.details + })); + }); + await insertChunked(tx, DeploymentEvents, rows); + } + + #logWarnings(warnings: ReducerWarning[]): void { + if (warnings.length === 0) { + return; + } + const byCode = new Map(); + for (const warning of warnings) { + byCode.set(warning.code, [...(byCode.get(warning.code) ?? []), warning]); + } + for (const [code, group] of byCode) { + this.#logger.warn({ event: code, count: group.length, samples: group.slice(0, 5) }); + } + } + + #requireId(accountIds: Map, address: string): number { + const id = accountIds.get(address); + if (id === undefined) { + throw new Error(`No interned account id for address ${address}`); + } + return id; + } + + #requireAddress(addressesById: Map, accountId: number): string { + const address = addressesById.get(accountId); + if (address === undefined) { + throw new Error(`No account row for id ${accountId}`); + } + return address; + } + + #requireDeploymentId(deploymentIds: Map, state: DeploymentAggState): number { + const id = deploymentIds.get(stateKey(state.key)); + if (id === undefined) { + throw new Error(`No deployment id for ${stateKey(state.key)}`); + } + return id; + } + + #requireGroupId(groupIds: Map, deploymentId: number, gseq: number): number { + const id = groupIds.get(`${deploymentId}/${gseq}`); + if (id === undefined) { + throw new Error(`No deployment group id for deployment ${deploymentId} gseq ${gseq}`); + } + return id; + } +} + +function toNormalizedResource(resource: typeof DeploymentGroupResources.$inferSelect): NormalizedResource { + return { + count: resource.count, + cpuUnits: resource.cpuUnits, + gpuUnits: resource.gpuUnits, + gpuVendor: resource.gpuVendor, + gpuModel: resource.gpuModel, + memoryBytes: resource.memoryBytes, + ephemeralStorageBytes: resource.ephemeralStorageBytes, + persistentStorageBytes: resource.persistentStorageBytes, + price: resource.price, + priceDenom: resource.priceDenom + }; +} + +/** Postgres normalizes numeric literals (e.g. strips leading zeros), so dseq comparisons go through one canonical form. */ +function normalizeDseq(dseq: string): string { + return BigInt(dseq).toString(); +} + +/** The (interned owner account id, canonical dseq) pair that keys a deployment across the load and flush maps. */ +function ownerDseqKey(ownerAccountId: number, dseq: string): string { + return `${ownerAccountId}/${normalizeDseq(dseq)}`; +} + +function compareDseq(a: string, b: string): number { + const left = BigInt(a); + const right = BigInt(b); + return left < right ? -1 : left > right ? 1 : 0; +} diff --git a/apps/chain-indexer/src/akash/dec.spec.ts b/apps/chain-indexer/src/akash/dec.spec.ts new file mode 100644 index 0000000000..81fa65abcb --- /dev/null +++ b/apps/chain-indexer/src/akash/dec.spec.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from "vitest"; + +import { DEC_ONE, decCeilInt, decFromInt, decFromString, decMul, decMulInt, decQuo, decToString, decTruncateInt, minBigInt } from "@src/akash/dec"; + +describe("dec", () => { + describe("decFromString", () => { + it("parses integer coin amounts", () => { + expect(decFromString("1000")).toBe(1000n * DEC_ONE); + }); + + it("parses fractional DecCoin amounts", () => { + expect(decFromString("1.5")).toBe(1_500_000_000_000_000_000n); + }); + + it("parses postgres numeric(38,18) output with full fractional padding", () => { + expect(decFromString("2.500000000000000000")).toBe(2_500_000_000_000_000_000n); + }); + + it("parses negative values", () => { + expect(decFromString("-0.5")).toBe(-500_000_000_000_000_000n); + }); + + it("rejects malformed strings", () => { + expect(() => decFromString("1e5")).toThrow("Invalid decimal string"); + expect(() => decFromString("")).toThrow("Invalid decimal string"); + }); + + it("rejects more than 18 fractional digits", () => { + expect(() => decFromString("1.0000000000000000001")).toThrow("18 fractional digits"); + }); + }); + + describe("decToString", () => { + it("round-trips integers and fractions", () => { + expect(decToString(decFromString("1000"))).toBe("1000"); + expect(decToString(decFromString("1.5"))).toBe("1.5"); + expect(decToString(decFromString("-0.5"))).toBe("-0.5"); + }); + + it("keeps full 18-digit precision", () => { + expect(decToString(1n)).toBe("0.000000000000000001"); + }); + }); + + describe("decQuo", () => { + it("rounds half away from zero at the 18th decimal like LegacyDec", () => { + expect(decQuo(decFromInt(1), decFromInt(3))).toBe(333_333_333_333_333_333n); + expect(decQuo(decFromInt(2), decFromInt(3))).toBe(666_666_666_666_666_667n); + }); + + it("rounds negative quotients half away from zero", () => { + expect(decQuo(decFromInt(-2), decFromInt(3))).toBe(-666_666_666_666_666_667n); + }); + + it("divides exactly when no remainder exists", () => { + expect(decQuo(decFromInt(10), decFromInt(4))).toBe(decFromString("2.5")); + }); + + it("throws on division by zero", () => { + expect(() => decQuo(DEC_ONE, 0n)).toThrow("Division by zero"); + }); + }); + + describe("decMul", () => { + it("multiplies with rounding at the 18th decimal", () => { + const oneThird = decQuo(decFromInt(1), decFromInt(3)); + expect(decMul(oneThird, decFromInt(3))).toBe(999_999_999_999_999_999n); + }); + + it("multiplies exact values without loss", () => { + expect(decMul(decFromString("1.5"), decFromString("2"))).toBe(decFromString("3")); + }); + }); + + describe("decMulInt", () => { + it("is exact for integer multipliers", () => { + expect(decMulInt(decFromString("0.000000000000000001"), 1_000_000_000_000_000_000n)).toBe(DEC_ONE); + }); + }); + + describe("decTruncateInt", () => { + it("truncates toward zero", () => { + expect(decTruncateInt(decFromString("2.9"))).toBe(2n); + expect(decTruncateInt(decFromString("2"))).toBe(2n); + expect(decTruncateInt(decFromString("-2.9"))).toBe(-2n); + }); + }); + + describe("decCeilInt", () => { + it("rounds up any fractional part", () => { + expect(decCeilInt(decFromString("2.000000000000000001"))).toBe(3n); + expect(decCeilInt(decFromString("2"))).toBe(2n); + }); + + it("ceils negatives toward positive infinity", () => { + expect(decCeilInt(decFromString("-2.5"))).toBe(-2n); + expect(decCeilInt(decFromString("-2"))).toBe(-2n); + }); + }); + + describe("minBigInt", () => { + it("returns the smaller value", () => { + expect(minBigInt(3n, 5n)).toBe(3n); + expect(minBigInt(5n, 3n)).toBe(3n); + }); + }); +}); diff --git a/apps/chain-indexer/src/akash/dec.ts b/apps/chain-indexer/src/akash/dec.ts new file mode 100644 index 0000000000..923c3ac317 --- /dev/null +++ b/apps/chain-indexer/src/akash/dec.ts @@ -0,0 +1,75 @@ +/** + * Fixed-point decimal math mirroring cosmos-sdk's LegacyDec: values are bigint atomics at 10^-18 + * scale. Escrow settlement must reproduce the chain's arithmetic exactly, which JS floats cannot + * (the legacy indexer's DOUBLE drift is the bug being fixed) and no decimal library replicates + * LegacyDec's two-step truncate-then-round quotient, so the four operations the keeper uses are + * implemented here directly. + */ +export const DEC_ONE = 10n ** 18n; + +const SQUARED_PRECISION = DEC_ONE * DEC_ONE; + +export function decFromString(value: string): bigint { + const match = /^(-?)(\d+)(?:\.(\d+))?$/.exec(value.trim()); + if (!match) throw new Error(`Invalid decimal string: ${value}`); + const [, sign, integerPart, fractionalPart = ""] = match; + if (fractionalPart.length > 18) throw new Error(`Decimal exceeds 18 fractional digits: ${value}`); + const atomics = BigInt(integerPart) * DEC_ONE + BigInt(fractionalPart.padEnd(18, "0")); + return sign === "-" ? -atomics : atomics; +} + +export function decFromInt(value: bigint | number): bigint { + return BigInt(value) * DEC_ONE; +} + +export function decToString(atomics: bigint): string { + const sign = atomics < 0n ? "-" : ""; + const abs = atomics < 0n ? -atomics : atomics; + const integerPart = abs / DEC_ONE; + const fractionalPart = (abs % DEC_ONE).toString().padStart(18, "0").replace(/0+$/, ""); + return `${sign}${integerPart}${fractionalPart ? `.${fractionalPart}` : ""}`; +} + +/** LegacyDec chopPrecisionAndRound: divide by 10^18 rounding half away from zero. */ +function chopPrecisionAndRound(value: bigint): bigint { + const negative = value < 0n; + const abs = negative ? -value : value; + const quotient = abs / DEC_ONE; + const remainder = abs % DEC_ONE; + const rounded = remainder * 2n >= DEC_ONE ? quotient + 1n : quotient; + return negative ? -rounded : rounded; +} + +export function decMul(a: bigint, b: bigint): bigint { + return chopPrecisionAndRound(a * b); +} + +export function decMulInt(a: bigint, b: bigint): bigint { + return a * b; +} + +/** + * LegacyDec Quo: scale the numerator by 10^36, truncate-divide by the denominator, then chop back + * one precision with rounding. The intermediate truncation is part of the chain's semantics, so the + * two steps are kept distinct instead of rounding a single 10^18-scaled quotient. + */ +export function decQuo(a: bigint, b: bigint): bigint { + if (b === 0n) throw new Error("Division by zero"); + return chopPrecisionAndRound((a * SQUARED_PRECISION) / b); +} + +/** Truncate toward zero to a whole integer (not atomics), matching LegacyDec TruncateInt. */ +export function decTruncateInt(atomics: bigint): bigint { + return atomics / DEC_ONE; +} + +/** Smallest integer (not atomics) greater than or equal to the value, matching LegacyDec Ceil for positive values. */ +export function decCeilInt(atomics: bigint): bigint { + const quotient = atomics / DEC_ONE; + const remainder = atomics % DEC_ONE; + return remainder > 0n ? quotient + 1n : quotient; +} + +export function minBigInt(a: bigint, b: bigint): bigint { + return a < b ? a : b; +} diff --git a/apps/chain-indexer/src/akash/denom.ts b/apps/chain-indexer/src/akash/denom.ts new file mode 100644 index 0000000000..84227d9441 --- /dev/null +++ b/apps/chain-indexer/src/akash/denom.ts @@ -0,0 +1,16 @@ +/** The IBC denoms deployments are funded with, mapped to their base denom (mirrors the legacy indexer's mapping). */ +const DENOM_MAPPING = new Map([ + ["uakt", "uakt"], + ["uact", "uact"], + ["ibc/028CD1864059EEFB48A6048376165318E3E82C234390AE5A6D7B22001725B06E", "uusdc"], + ["ibc/170C677610AC31DF0904FFE09CD3B5C657492170E7E52372E48756B71E56F2F1", "uusdc"] +]); + +/** + * Unknown denoms are stored raw instead of throwing (the legacy indexer aborts the block), so a new + * funding denom degrades to an unmapped row rather than halting ingestion. + */ +export function normalizeDenom(denom: string): { denom: string; known: boolean } { + const mapped = DENOM_MAPPING.get(denom); + return mapped ? { denom: mapped, known: true } : { denom, known: false }; +} diff --git a/apps/chain-indexer/src/akash/deployment-reducer.spec.ts b/apps/chain-indexer/src/akash/deployment-reducer.spec.ts new file mode 100644 index 0000000000..6ebc1dd3dd --- /dev/null +++ b/apps/chain-indexer/src/akash/deployment-reducer.spec.ts @@ -0,0 +1,324 @@ +import { describe, expect, it } from "vitest"; + +import type { AkashBlockChanges, AkashChangeBody, NormalizedGroup } from "@src/akash/akash-changes"; +import { decFromInt } from "@src/akash/dec"; +import type { DeploymentAggState } from "@src/akash/deployment-reducer"; +import { applyBlockChanges, stateKey } from "@src/akash/deployment-reducer"; + +const OWNER = "akash1owner"; +const PROVIDER = "akash1prov"; +const KEY = { owner: OWNER, dseq: "42" }; +const LEASE_KEY = { ...KEY, gseq: 1, oseq: 1, bseq: 0, provider: PROVIDER }; +const BLOCK_TIME = new Date("2026-08-13T00:00:00Z"); + +describe("applyBlockChanges", () => { + it("creates a deployment whose totals are the sum of its group resources", () => { + const { states } = setup(); + + applyBlockChanges(states, block(100, [create({ groups: twoGroups() })])); + + const state = get(states); + expect(state.deposit).toBe(5000000n); + expect(state.balance).toBe(decFromInt(5000000)); + expect(state.denom).toBe("uakt"); + expect(state.cpuUnits).toBe(1000 * 2 + 500 * 3); + expect(state.gpuUnits).toBe(0 * 2 + 1 * 3); + expect(state.memoryBytes).toBe(1024 * 2 + 2048 * 3); + expect(state.ephemeralStorageBytes).toBe(100 * 2 + 200 * 3); + expect(state.persistentStorageBytes).toBe(50 * 2 + 0 * 3); + expect(state.groups).toHaveLength(2); + expect(state.events).toEqual([{ height: 100, ordinal: 0, txIndex: 0, msgIndex: 0, type: "created", details: { deposit: "5000000", denom: "uakt" } }]); + }); + + it("maps an ibc funding denom and warns on an unknown one instead of throwing", () => { + const { states } = setup(); + + const warnings = applyBlockChanges( + states, + block(100, [ + create({ denom: "ibc/170C677610AC31DF0904FFE09CD3B5C657492170E7E52372E48756B71E56F2F1" }), + { ...create({ denom: "ibc/deadbeef" }), key: { owner: OWNER, dseq: "43" } } + ]) + ); + + expect(get(states).denom).toBe("uusdc"); + expect(states.get(`${OWNER}/43`)?.denom).toBe("ibc/deadbeef"); + expect(warnings).toEqual([{ code: "AKASH_UNKNOWN_DENOM", kind: "deploymentCreated", owner: OWNER, dseq: "43", height: 100 }]); + }); + + it("runs the full lifecycle: create, bid, lease, withdraw, close", () => { + const { states } = setup(); + + applyBlockChanges(states, block(100, [create({}), bidCreated("10")])); + applyBlockChanges(states, block(110, [leaseCreated()])); + applyBlockChanges(states, block(150, [{ kind: "leaseWithdrawn", key: LEASE_KEY }])); + applyBlockChanges(states, block(200, [{ kind: "deploymentClosed", key: KEY }])); + + const state = get(states); + const lease = state.leases[0]; + expect(lease.price).toBe(decFromInt(10)); + expect(lease.cpuUnits).toBe(2000); + expect(lease.withdrawn).toBe(decFromInt(90 * 10)); + expect(lease.closedHeight).toBe(200); + expect(lease.closedAt).toBe(BLOCK_TIME); + expect(state.withdrawn).toBe(decFromInt(900)); + expect(state.balance).toBe(decFromInt(5000000 - 900)); + expect(state.lastWithdrawHeight).toBe(200); + expect(state.closedHeight).toBe(200); + expect(state.closeReason).toBe("close_message"); + expect(state.bids[0].state).toBe("closed"); + expect(state.events.map(event => event.type)).toEqual(["created", "bid_created", "lease_created", "lease_withdrawn", "closed"]); + }); + + it("computes the lease predicted close height from the bid price and re-predicts on deposit", () => { + const { states } = setup(); + + applyBlockChanges(states, block(100, [create({ deposit: "1000" }), bidCreated("10")])); + applyBlockChanges(states, block(110, [leaseCreated()])); + + expect(get(states).leases[0].predictedClosedHeight).toBe(110n + 100n); + + applyBlockChanges(states, block(120, [{ kind: "deploymentDeposited", key: KEY, amount: "1000", depositor: "akash1grantee" }])); + + const state = get(states); + expect(state.deposit).toBe(2000n); + expect(state.balance).toBe(decFromInt(2000)); + expect(state.leases[0].predictedClosedHeight).toBe(110n + 200n); + expect(state.events.at(-1)).toMatchObject({ type: "deposited", details: { amount: "1000", denom: "uakt", depositor: "akash1grantee" } }); + }); + + it("closes everything with reason overdrawn when a settlement exhausts the balance", () => { + const { states } = setup(); + + applyBlockChanges(states, block(100, [create({ deposit: "1000" }), bidCreated("10")])); + applyBlockChanges(states, block(110, [leaseCreated()])); + applyBlockChanges(states, block(300, [{ kind: "leaseWithdrawn", key: LEASE_KEY }])); + + const state = get(states); + expect(state.balance).toBe(0n); + expect(state.withdrawn).toBe(decFromInt(1000)); + expect(state.closedHeight).toBe(300); + expect(state.closeReason).toBe("overdrawn"); + expect(state.leases[0].closedHeight).toBe(300); + expect(state.leases[0].withdrawn).toBe(decFromInt(1000)); + expect(state.bids[0].state).toBe("closed"); + expect(state.events.map(event => event.type)).toEqual(["created", "bid_created", "lease_created", "closed", "lease_withdrawn"]); + }); + + it("keeps the deployment open when leases close and re-predicts the remaining ones", () => { + const { states } = setup(); + const secondLease = { ...LEASE_KEY, gseq: 2 }; + + applyBlockChanges(states, block(100, [create({ groups: twoGroups() }), bidCreated("10"), { ...bidCreated("30"), key: secondLease }])); + applyBlockChanges(states, block(110, [leaseCreated(), { kind: "leaseCreated", key: secondLease }])); + applyBlockChanges(states, block(120, [{ kind: "leaseClosed", key: secondLease }])); + + let state = get(states); + expect(state.closedHeight).toBeNull(); + expect(state.leases.find(lease => lease.gseq === 2)).toMatchObject({ closedHeight: 120, withdrawn: decFromInt(300), balance: 0n }); + expect(state.balance).toBe(decFromInt(5000000 - 400)); + expect(state.leases[0].predictedClosedHeight).toBe(120n + 499960n); + + applyBlockChanges(states, block(130, [{ kind: "leaseClosed", key: LEASE_KEY }])); + + state = get(states); + expect(state.closedHeight).toBeNull(); + expect(state.leases[0].closedHeight).toBe(130); + expect(state.withdrawn).toBe(decFromInt(300 + 200)); + }); + + it("truncates payouts to whole units and refunds the fraction to the deployment on lease close", () => { + const { states } = setup(); + + applyBlockChanges(states, block(100, [create({ deposit: "500000" }), bidCreated("2.349334")])); + applyBlockChanges(states, block(110, [leaseCreated()])); + applyBlockChanges(states, block(122, [{ kind: "leaseClosed", key: LEASE_KEY }])); + + const state = get(states); + expect(state.leases[0].withdrawn).toBe(decFromInt(28)); + expect(state.leases[0].balance).toBe(0n); + expect(state.withdrawn).toBe(decFromInt(28)); + expect(state.balance).toBe(decFromInt(500000 - 28)); + }); + + it("skips a duplicate block per deployment via the watermark but applies later blocks", () => { + const { states } = setup(); + + applyBlockChanges(states, block(100, [create({}), bidCreated("10")])); + applyBlockChanges(states, block(110, [leaseCreated()])); + const snapshot = JSON.stringify(get(states), stringifyBigInt); + + applyBlockChanges(states, block(110, [leaseCreated()])); + + expect(JSON.stringify(get(states), stringifyBigInt)).toBe(snapshot); + expect(get(states).leases).toHaveLength(1); + + applyBlockChanges(states, block(150, [{ kind: "leaseWithdrawn", key: LEASE_KEY }])); + expect(get(states).lastWithdrawHeight).toBe(150); + }); + + it("reports orphan references without mutating state", () => { + const { states } = setup(); + + const warnings = applyBlockChanges(states, block(100, [{ kind: "deploymentDeposited", key: KEY, amount: "5", depositor: null }])); + + expect(states.size).toBe(0); + expect(warnings).toEqual([{ code: "AKASH_ORPHAN_REFERENCE", kind: "deploymentDeposited", owner: OWNER, dseq: "42", height: 100 }]); + }); + + it("applies close-event fallbacks with settlement, only when not already closed", () => { + const { states } = setup(); + + applyBlockChanges(states, block(100, [create({ deposit: "10000" }), bidCreated("10")])); + applyBlockChanges(states, block(110, [leaseCreated()])); + applyBlockChanges(states, block(120, [{ kind: "leaseClosedEvent", key: KEY, gseq: 1, oseq: 1, bseq: null, provider: PROVIDER }])); + + let state = get(states); + expect(state.leases[0].closedHeight).toBe(120); + expect(state.leases[0].withdrawn).toBe(decFromInt(100)); + expect(state.closedHeight).toBeNull(); + + applyBlockChanges(states, block(130, [{ kind: "deploymentClosedEvent", key: KEY }])); + state = get(states); + expect(state.closedHeight).toBe(130); + expect(state.closeReason).toBe("close_event"); + + applyBlockChanges(states, block(140, [{ kind: "deploymentClosedEvent", key: KEY }])); + expect(get(states).closedHeight).toBe(130); + }); + + it("tracks group lifecycle transitions without reopening a closed group", () => { + const { states } = setup(); + + applyBlockChanges(states, block(100, [create({})])); + applyBlockChanges(states, block(110, [{ kind: "groupPaused", key: KEY, gseq: 1 }])); + expect(get(states).groups[0].state).toBe("paused"); + + applyBlockChanges(states, block(120, [{ kind: "groupClosed", key: KEY, gseq: 1 }])); + expect(get(states).groups[0]).toMatchObject({ state: "closed", closedHeight: 120 }); + + applyBlockChanges(states, block(130, [{ kind: "groupStarted", key: KEY, gseq: 1 }])); + expect(get(states).groups[0].state).toBe("closed"); + }); + + it("assigns sequential event ordinals within a block", () => { + const { states } = setup(); + + applyBlockChanges(states, block(100, [create({}), { kind: "deploymentUpdated", key: KEY }, bidCreated("10")])); + + expect(get(states).events.map(event => [event.type, event.ordinal])).toEqual([ + ["created", 0], + ["updated", 1], + ["bid_created", 2] + ]); + }); + + it("tolerates a zero-rate lease from an unparseable bid price without dividing by zero", () => { + const { states } = setup(); + + applyBlockChanges(states, block(100, [create({ deposit: "1000" }), bidCreated("not-a-number")])); + applyBlockChanges(states, block(110, [leaseCreated()])); + + expect(() => applyBlockChanges(states, block(150, [{ kind: "leaseWithdrawn", key: LEASE_KEY }]))).not.toThrow(); + + const state = get(states); + expect(state.leases[0].price).toBe(0n); + expect(state.leases[0].withdrawn).toBe(0n); + expect(state.balance).toBe(decFromInt(1000)); + expect(state.closedHeight).toBeNull(); + }); + + it("warns and creates a zero-rate lease when the matching bid is missing", () => { + const { states } = setup(); + + applyBlockChanges(states, block(100, [create({})])); + const warnings = applyBlockChanges(states, block(110, [leaseCreated()])); + + expect(warnings).toEqual([{ code: "AKASH_ORPHAN_REFERENCE", kind: "leaseCreated", owner: OWNER, dseq: "42", height: 110 }]); + expect(get(states).leases).toHaveLength(1); + expect(get(states).leases[0].price).toBe(0n); + }); + + it("closes the order's other open bids when a lease is created", () => { + const { states } = setup(); + const rivalBid = { ...LEASE_KEY, provider: "akash1rival" }; + + applyBlockChanges(states, block(100, [create({}), bidCreated("10"), { kind: "bidCreated", key: rivalBid, price: "9", priceDenom: "uakt" }])); + applyBlockChanges(states, block(110, [leaseCreated()])); + + const state = get(states); + expect(state.bids.find(bid => bid.provider === PROVIDER)).toMatchObject({ state: "active" }); + expect(state.bids.find(bid => bid.provider === "akash1rival")).toMatchObject({ state: "closed", closedHeight: 110 }); + }); + + function setup() { + return { states: new Map() }; + } + + function get(states: Map): DeploymentAggState { + const state = states.get(stateKey(KEY)); + if (!state) { + throw new Error("deployment state missing"); + } + return state; + } + + function block(height: number, bodies: AkashChangeBody[]): AkashBlockChanges { + return { + height, + datetime: BLOCK_TIME, + changes: bodies.map((body, index) => ({ ...body, txIndex: 0, msgIndex: index })) + }; + } + + function create(input: { deposit?: string; denom?: string; groups?: NormalizedGroup[] }): AkashChangeBody { + return { + kind: "deploymentCreated", + key: KEY, + denom: input.denom ?? "uakt", + deposit: input.deposit ?? "5000000", + depositor: null, + groups: input.groups ?? [group(1, { cpuUnits: 1000, count: 2 })] + }; + } + + function bidCreated(price: string): AkashChangeBody { + return { kind: "bidCreated", key: LEASE_KEY, price, priceDenom: "uakt" }; + } + + function leaseCreated(): AkashChangeBody { + return { kind: "leaseCreated", key: LEASE_KEY }; + } + + function twoGroups(): NormalizedGroup[] { + return [ + group(1, { cpuUnits: 1000, gpuUnits: 0, memoryBytes: 1024, ephemeralStorageBytes: 100, persistentStorageBytes: 50, count: 2 }), + group(2, { cpuUnits: 500, gpuUnits: 1, memoryBytes: 2048, ephemeralStorageBytes: 200, persistentStorageBytes: 0, count: 3 }) + ]; + } + + function group(gseq: number, resource: Partial): NormalizedGroup { + return { + gseq, + resources: [ + { + count: 1, + cpuUnits: 0, + gpuUnits: 0, + gpuVendor: null, + gpuModel: null, + memoryBytes: 0, + ephemeralStorageBytes: 0, + persistentStorageBytes: 0, + price: "1", + priceDenom: "uakt", + ...resource + } + ] + }; + } + + function stringifyBigInt(_: string, value: unknown): unknown { + return typeof value === "bigint" ? value.toString() : value; + } +}); diff --git a/apps/chain-indexer/src/akash/deployment-reducer.ts b/apps/chain-indexer/src/akash/deployment-reducer.ts new file mode 100644 index 0000000000..59809b9385 --- /dev/null +++ b/apps/chain-indexer/src/akash/deployment-reducer.ts @@ -0,0 +1,584 @@ +import type { AkashBlockChanges, AkashChange, DeploymentKey, LeaseSlot, NormalizedGroup, NormalizedResource } from "@src/akash/akash-changes"; +import { decCeilInt, decFromInt, decFromString, decQuo, decToString, decTruncateInt } from "@src/akash/dec"; +import { normalizeDenom } from "@src/akash/denom"; +import { settle, sumLeaseRate } from "@src/akash/settlement"; +import type { bidState, deploymentCloseReason, deploymentEventType, groupState } from "@src/db/schema"; + +export type DeploymentCloseReason = (typeof deploymentCloseReason.enumValues)[number]; +export type DeploymentEventType = (typeof deploymentEventType.enumValues)[number]; +export type GroupStateValue = (typeof groupState.enumValues)[number]; +export type BidStateValue = (typeof bidState.enumValues)[number]; + +export interface ResourceTotals { + cpuUnits: number; + gpuUnits: number; + memoryBytes: number; + ephemeralStorageBytes: number; + persistentStorageBytes: number; +} + +export interface GroupAggState { + gseq: number; + state: GroupStateValue; + closedHeight: number | null; + resources: NormalizedResource[]; +} + +export interface BidAggState { + gseq: number; + oseq: number; + bseq: number; + provider: string; + price: bigint; + denom: string; + state: BidStateValue; + createdHeight: number; + closedHeight: number | null; +} + +export interface LeaseAggState extends ResourceTotals { + gseq: number; + oseq: number; + bseq: number; + provider: string; + price: bigint; + denom: string; + /** Accrued-but-unwithdrawn earnings, mirroring the on-chain payment balance. */ + balance: bigint; + /** Paid-out total; the chain truncates every payout to whole units, so this is always integral. */ + withdrawn: bigint; + predictedClosedHeight: bigint; + createdHeight: number; + createdAt: Date; + closedHeight: number | null; + closedAt: Date | null; +} + +export interface DeploymentEventDraft { + height: number; + ordinal: number; + txIndex: number | null; + msgIndex: number | null; + type: DeploymentEventType; + details: Record | null; +} + +export interface DeploymentAggState extends ResourceTotals { + key: DeploymentKey; + denom: string; + deposit: bigint; + balance: bigint; + withdrawn: bigint; + lastWithdrawHeight: number | null; + lastProcessedHeight: number; + createdHeight: number; + createdAt: Date; + closedHeight: number | null; + closedAt: Date | null; + closeReason: DeploymentCloseReason | null; + groups: GroupAggState[]; + bids: BidAggState[]; + leases: LeaseAggState[]; + events: DeploymentEventDraft[]; + isNew: boolean; + touched: boolean; +} + +export interface ReducerWarning { + code: "AKASH_ORPHAN_REFERENCE" | "AKASH_UNKNOWN_DENOM"; + kind: AkashChange["kind"]; + owner: string; + dseq: string; + height: number; +} + +export function stateKey(key: DeploymentKey): string { + return `${key.owner}/${key.dseq}`; +} + +/** + * Applies one block's derived changes to the in-memory deployment states, porting the legacy + * indexer's handler semantics onto the current keeper's exact escrow math. Blocks must be applied in + * ascending height order. A block at or below a deployment's `lastProcessedHeight` watermark is a + * duplicate commit (replay or an overlapping writer) and is skipped for that deployment, which keeps + * the read-modify-write escrow state idempotent; both runners commit strictly in order, so an + * older-than-watermark block can never carry unseen changes. + */ +export function applyBlockChanges(states: Map, block: AkashBlockChanges): ReducerWarning[] { + const warnings: ReducerWarning[] = []; + const decided = new Map(); + + for (const change of block.changes) { + const key = stateKey(change.key); + + if (change.kind === "deploymentCreated") { + if (shouldApply(decided, states.get(key), block.height, key)) { + createDeployment(states, change, block, warnings); + } + continue; + } + + const state = states.get(key); + if (!state) { + warnings.push({ code: "AKASH_ORPHAN_REFERENCE", kind: change.kind, owner: change.key.owner, dseq: change.key.dseq, height: block.height }); + continue; + } + if (!shouldApply(decided, state, block.height, key)) { + continue; + } + + applyChange(state, change, block, warnings); + } + + for (const [key, applied] of decided) { + const state = states.get(key); + if (state && applied) { + state.lastProcessedHeight = block.height; + state.touched = true; + } + } + + return warnings; +} + +/** The skip decision is made once per deployment per block, so a deployment created earlier in the same block still receives its later changes. */ +function shouldApply(decided: Map, state: DeploymentAggState | undefined, height: number, key: string): boolean { + const existing = decided.get(key); + if (existing !== undefined) { + return existing; + } + const applies = !state || height > state.lastProcessedHeight; + decided.set(key, applies); + return applies; +} + +function applyChange(state: DeploymentAggState, change: AkashChange, block: AkashBlockChanges, warnings: ReducerWarning[]): void { + switch (change.kind) { + case "deploymentDeposited": + return applyDeposit(state, change, block); + case "deploymentUpdated": + return addEvent(state, block, change, "updated", null); + case "deploymentClosed": + return applyDeploymentClose(state, change, block, "close_message"); + case "deploymentClosedEvent": + return applyDeploymentCloseEvent(state, change, block); + case "groupClosed": + return applyGroupChange(state, change, block, "closed", "group_closed"); + case "groupPaused": + return applyGroupChange(state, change, block, "paused", "group_paused"); + case "groupStarted": + return applyGroupChange(state, change, block, "open", "group_started"); + case "bidCreated": + return applyBidCreated(state, change, block); + case "bidClosed": + return applyBidClosed(state, change, block); + case "leaseCreated": + return applyLeaseCreated(state, change, block, warnings); + case "leaseClosed": + return applyLeaseClosed(state, change, block); + case "leaseWithdrawn": + return applyLeaseWithdrawn(state, change, block, warnings); + case "leaseClosedEvent": + return applyLeaseClosedEvent(state, change, block); + } +} + +function createDeployment( + states: Map, + change: Extract, + block: AkashBlockChanges, + warnings: ReducerWarning[] +): void { + const { denom, known } = normalizeDenom(change.denom); + if (!known) { + warnings.push({ code: "AKASH_UNKNOWN_DENOM", kind: change.kind, owner: change.key.owner, dseq: change.key.dseq, height: block.height }); + } + + const state: DeploymentAggState = { + key: change.key, + denom, + deposit: BigInt(change.deposit), + balance: decFromString(change.deposit), + withdrawn: 0n, + lastWithdrawHeight: null, + lastProcessedHeight: 0, + createdHeight: block.height, + createdAt: block.datetime, + closedHeight: null, + closedAt: null, + closeReason: null, + ...sumGroupTotals(change.groups), + groups: change.groups.map(group => ({ gseq: group.gseq, state: "open", closedHeight: null, resources: group.resources })), + bids: [], + leases: [], + events: [], + isNew: true, + touched: true + }; + + states.set(stateKey(change.key), state); + addEvent(state, block, change, "created", { deposit: change.deposit, denom, ...(change.depositor ? { depositor: change.depositor } : {}) }); +} + +function applyDeposit(state: DeploymentAggState, change: Extract, block: AkashBlockChanges): void { + state.deposit += BigInt(change.amount); + state.balance += decFromString(change.amount); + + const openLeases = state.leases.filter(lease => lease.closedHeight === null); + const blockRate = sumLeaseRate(openLeases); + for (const lease of openLeases) { + lease.predictedClosedHeight = predictClosedHeight(state.lastWithdrawHeight ?? lease.createdHeight, state.balance, blockRate); + } + + addEvent(state, block, change, "deposited", { amount: change.amount, denom: state.denom, ...(change.depositor ? { depositor: change.depositor } : {}) }); +} + +function applyDeploymentClose(state: DeploymentAggState, change: AkashChange, block: AkashBlockChanges, reason: DeploymentCloseReason): void { + if (state.closedHeight !== null) { + return; + } + settleState(state, block, change); + if (state.closedHeight !== null) { + return; + } + closeDeployment(state, block, reason); + addEvent(state, block, change, "closed", { reason }); +} + +/** + * Side-effect closes (group close, authz revoke) arrive as chain events rather than messages. The + * chain settles the escrow account when it closes, so the fallback settles too — a deliberate fix + * over the legacy indexer, which only stamped the height and let balances drift. + */ +function applyDeploymentCloseEvent(state: DeploymentAggState, change: AkashChange, block: AkashBlockChanges): void { + if (state.closedHeight !== null) { + return; + } + applyDeploymentClose(state, change, block, "close_event"); +} + +function applyGroupChange( + state: DeploymentAggState, + change: Extract, + block: AkashBlockChanges, + groupStateValue: GroupStateValue, + eventType: DeploymentEventType +): void { + const group = state.groups.find(candidate => candidate.gseq === change.gseq); + if (!group || group.state === "closed") { + return; + } + group.state = groupStateValue; + if (groupStateValue === "closed") { + group.closedHeight = block.height; + } + addEvent(state, block, change, eventType, { gseq: change.gseq }); +} + +function applyBidCreated(state: DeploymentAggState, change: Extract, block: AkashBlockChanges): void { + const bid: BidAggState = { + gseq: change.key.gseq, + oseq: change.key.oseq, + bseq: change.key.bseq, + provider: change.key.provider, + price: parsePrice(change.price), + denom: change.priceDenom, + state: "open", + createdHeight: block.height, + closedHeight: null + }; + + const existingIndex = state.bids.findIndex(candidate => sameLeaseKey(candidate, bid)); + if (existingIndex >= 0) { + state.bids[existingIndex] = bid; + } else { + state.bids.push(bid); + } + + addEvent(state, block, change, "bid_created", bidEventDetails(change.key, change.price, change.priceDenom)); +} + +function applyBidClosed(state: DeploymentAggState, change: Extract, block: AkashBlockChanges): void { + const lease = findOpenLease(state, change.key); + if (lease) { + closeLease(state, lease, change, block); + } + + const bid = state.bids.find(candidate => sameLeaseKey(candidate, change.key)); + if (bid && bid.state !== "closed") { + bid.state = "closed"; + bid.closedHeight = block.height; + } + addEvent(state, block, change, "bid_closed", bidEventDetails(change.key)); +} + +function applyLeaseCreated( + state: DeploymentAggState, + change: Extract, + block: AkashBlockChanges, + warnings: ReducerWarning[] +): void { + const bid = state.bids.find(candidate => sameLeaseKey(candidate, change.key)); + const group = state.groups.find(candidate => candidate.gseq === change.key.gseq); + if (!bid || !group) { + warnings.push({ code: "AKASH_ORPHAN_REFERENCE", kind: change.kind, owner: change.key.owner, dseq: change.key.dseq, height: block.height }); + } + + const { blockRate } = settleState(state, block, change); + const price = bid?.price ?? 0n; + const predicted = predictClosedHeight(block.height, state.balance, blockRate + price); + + const lease: LeaseAggState = { + gseq: change.key.gseq, + oseq: change.key.oseq, + bseq: change.key.bseq, + provider: change.key.provider, + price, + denom: state.denom, + balance: 0n, + withdrawn: 0n, + predictedClosedHeight: predicted, + createdHeight: block.height, + createdAt: block.datetime, + closedHeight: null, + closedAt: null, + ...sumResourceTotals(group?.resources ?? []) + }; + state.leases.push(lease); + + for (const openLease of state.leases.filter(candidate => candidate.closedHeight === null)) { + openLease.predictedClosedHeight = predicted; + } + + if (bid) { + bid.state = "active"; + } + + closeLosingBids(state, change.key, block.height); + + addEvent(state, block, change, "lease_created", bidEventDetails(change.key, bid ? decToString(bid.price) : undefined, bid?.denom)); +} + +function applyLeaseClosed(state: DeploymentAggState, change: Extract, block: AkashBlockChanges): void { + const lease = findOpenLease(state, change.key); + if (!lease) { + return; + } + closeLease(state, lease, change, block); + addEvent(state, block, change, "lease_closed", bidEventDetails(change.key)); +} + +function applyLeaseWithdrawn( + state: DeploymentAggState, + change: Extract, + block: AkashBlockChanges, + warnings: ReducerWarning[] +): void { + const lease = state.leases.find(candidate => sameLeaseKey(candidate, change.key)); + if (!lease) { + warnings.push({ code: "AKASH_ORPHAN_REFERENCE", kind: change.kind, owner: change.key.owner, dseq: change.key.dseq, height: block.height }); + return; + } + settleState(state, block, change); + withdrawFromLease(state, lease); + addEvent(state, block, change, "lease_withdrawn", bidEventDetails(change.key)); +} + +/** A lease closed by the chain without a direct message (its group closed, the order was revoked). Skipped when the close was already applied by the triggering message. */ +function applyLeaseClosedEvent(state: DeploymentAggState, change: Extract, block: AkashBlockChanges): void { + const lease = state.leases.find( + candidate => + candidate.closedHeight === null && + candidate.gseq === change.gseq && + candidate.oseq === change.oseq && + candidate.provider === change.provider && + (change.bseq === null || candidate.bseq === change.bseq) + ); + if (!lease) { + return; + } + closeLease(state, lease, change, block); + addEvent(state, block, change, "lease_closed", { + gseq: change.gseq, + oseq: change.oseq, + bseq: change.bseq ?? lease.bseq, + provider: change.provider + }); +} + +/** + * Shared close path for lease-terminating changes: settle first, pay out and close the lease, then + * re-predict the remaining leases at the reduced block rate. Unlike the legacy indexer, the + * deployment stays open when its last lease closes — the chain keeps the escrow account alive, and + * actual deployment closes always arrive as a message, an overdraw, or a close event. + */ +function closeLease(state: DeploymentAggState, lease: LeaseAggState, change: AkashChange, block: AkashBlockChanges): void { + const { blockRate } = settleState(state, block, change); + + if (lease.closedHeight === null) { + lease.closedHeight = block.height; + lease.closedAt = block.datetime; + payOutClosedLease(state, lease); + } + + if (state.closedHeight !== null) { + return; + } + + const remainingRate = blockRate - lease.price; + for (const openLease of state.leases.filter(candidate => candidate.closedHeight === null)) { + openLease.predictedClosedHeight = predictClosedHeight(state.lastWithdrawHeight ?? openLease.createdHeight, state.balance, remainingRate); + } +} + +/** Runs the escrow settlement and, when it overdraws, records the chain's forced close of the deployment and every open lease. */ +function settleState(state: DeploymentAggState, block: AkashBlockChanges, change: AkashChange): { blockRate: bigint } { + const openLeases = state.leases.filter(lease => lease.closedHeight === null); + const { blockRate, overdrawn } = settle(state, openLeases, block.height); + + if (overdrawn) { + for (const lease of openLeases) { + lease.closedAt = block.datetime; + payOutClosedLease(state, lease); + } + state.closedAt = block.datetime; + state.closeReason = "overdrawn"; + closeOpenBids(state, block.height); + addEvent(state, block, change, "closed", { reason: "overdrawn" }); + } + + return { blockRate }; +} + +/** + * Mirrors the keeper's payout on withdraw: the whole-unit part of the accrued balance moves to the + * lease's withdrawn total, the fraction stays accrued until the lease closes. + */ +function withdrawFromLease(state: DeploymentAggState, lease: LeaseAggState): void { + const paid = decFromInt(decTruncateInt(lease.balance)); + lease.balance -= paid; + lease.withdrawn += paid; + state.withdrawn += paid; +} + +/** On lease close the keeper pays out the whole units and refunds the fractional remainder to the account funds. */ +function payOutClosedLease(state: DeploymentAggState, lease: LeaseAggState): void { + withdrawFromLease(state, lease); + state.balance += lease.balance; + lease.balance = 0n; +} + +function closeDeployment(state: DeploymentAggState, block: AkashBlockChanges, reason: DeploymentCloseReason): void { + for (const lease of state.leases) { + if (lease.closedHeight === null) { + lease.closedHeight = block.height; + lease.closedAt = block.datetime; + payOutClosedLease(state, lease); + } + } + closeOpenBids(state, block.height); + state.closedHeight = block.height; + state.closedAt = block.datetime; + state.closeReason = reason; +} + +/** The chain closes a deployment's open bids with it; the legacy indexer deleted bid rows instead, so this is state the rewrite adds. */ +function closeOpenBids(state: DeploymentAggState, height: number): void { + for (const bid of state.bids) { + if (bid.state !== "closed") { + bid.state = "closed"; + bid.closedHeight = height; + } + } +} + +/** Creating a lease matches and closes the order, so the chain closes every other still-open bid on the same (gseq, oseq). */ +function closeLosingBids(state: DeploymentAggState, winning: LeaseSlot, height: number): void { + for (const bid of state.bids) { + if (bid.state !== "closed" && bid.gseq === winning.gseq && bid.oseq === winning.oseq && !sameLeaseKey(bid, winning)) { + bid.state = "closed"; + bid.closedHeight = height; + } + } +} + +/** + * The legacy predicted-close formula, `base + ceil(balance / rate)`, on exact math. A zero rate means + * the balance never depletes; the prediction is pinned to the base height so draining queries treat + * the lease as expired rather than dividing by zero. + */ +function predictClosedHeight(baseHeight: number, balance: bigint, blockRate: bigint): bigint { + if (blockRate <= 0n) { + return BigInt(baseHeight); + } + return BigInt(baseHeight) + decCeilInt(decQuo(balance, blockRate)); +} + +function addEvent( + state: DeploymentAggState, + block: AkashBlockChanges, + change: AkashChange, + type: DeploymentEventType, + details: Record | null +): void { + const ordinal = state.events.filter(event => event.height === block.height).length; + state.events.push({ height: block.height, ordinal, txIndex: change.txIndex, msgIndex: change.msgIndex, type, details }); +} + +function bidEventDetails(key: LeaseSlot, price?: string, denom?: string): Record { + return { + gseq: key.gseq, + oseq: key.oseq, + bseq: key.bseq, + provider: key.provider, + ...(price !== undefined ? { price } : {}), + ...(denom !== undefined && denom !== "" ? { denom } : {}) + }; +} + +function findOpenLease(state: DeploymentAggState, key: LeaseSlot): LeaseAggState | undefined { + return state.leases.find(candidate => candidate.closedHeight === null && sameLeaseKey(candidate, key)); +} + +function sameLeaseKey(a: LeaseSlot, b: LeaseSlot): boolean { + return a.gseq === b.gseq && a.oseq === b.oseq && a.bseq === b.bseq && a.provider === b.provider; +} + +function sumGroupTotals(groups: NormalizedGroup[]): ResourceTotals { + return groups.map(group => sumResourceTotals(group.resources)).reduce(addTotals, emptyTotals()); +} + +function sumResourceTotals(resources: NormalizedResource[]): ResourceTotals { + return resources + .map(resource => ({ + cpuUnits: resource.cpuUnits * resource.count, + gpuUnits: resource.gpuUnits * resource.count, + memoryBytes: resource.memoryBytes * resource.count, + ephemeralStorageBytes: resource.ephemeralStorageBytes * resource.count, + persistentStorageBytes: resource.persistentStorageBytes * resource.count + })) + .reduce(addTotals, emptyTotals()); +} + +function addTotals(a: ResourceTotals, b: ResourceTotals): ResourceTotals { + return { + cpuUnits: a.cpuUnits + b.cpuUnits, + gpuUnits: a.gpuUnits + b.gpuUnits, + memoryBytes: a.memoryBytes + b.memoryBytes, + ephemeralStorageBytes: a.ephemeralStorageBytes + b.ephemeralStorageBytes, + persistentStorageBytes: a.persistentStorageBytes + b.persistentStorageBytes + }; +} + +function emptyTotals(): ResourceTotals { + return { cpuUnits: 0, gpuUnits: 0, memoryBytes: 0, ephemeralStorageBytes: 0, persistentStorageBytes: 0 }; +} + +/** Bid prices are integer coins through v1beta2 and DecCoin decimal strings from v1beta3; a malformed price degrades to zero like the legacy `?? 0`. */ +function parsePrice(price: string): bigint { + try { + return decFromString(price); + } catch { + return 0n; + } +} diff --git a/apps/chain-indexer/src/akash/json.spec.ts b/apps/chain-indexer/src/akash/json.spec.ts new file mode 100644 index 0000000000..1ee49ef8f0 --- /dev/null +++ b/apps/chain-indexer/src/akash/json.spec.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from "vitest"; + +import { asInteger } from "@src/akash/json"; + +describe("asInteger", () => { + it("accepts non-negative safe integers and unsigned digit strings", () => { + expect(asInteger(0)).toBe(0); + expect(asInteger(42)).toBe(42); + expect(asInteger("42")).toBe(42); + }); + + it("rejects negative numbers and negative strings", () => { + expect(asInteger(-1)).toBeNull(); + expect(asInteger("-1")).toBeNull(); + }); + + it("rejects unsafe integers and non-integers", () => { + expect(asInteger(2 ** 53)).toBeNull(); + expect(asInteger(1.5)).toBeNull(); + expect(asInteger("99999999999999999999")).toBeNull(); + }); + + it("rejects non-numeric values", () => { + expect(asInteger("abc")).toBeNull(); + expect(asInteger(null)).toBeNull(); + expect(asInteger(undefined)).toBeNull(); + }); +}); diff --git a/apps/chain-indexer/src/akash/json.ts b/apps/chain-indexer/src/akash/json.ts new file mode 100644 index 0000000000..360239ed32 --- /dev/null +++ b/apps/chain-indexer/src/akash/json.ts @@ -0,0 +1,18 @@ +export function asRecord(value: unknown): Record | null { + return typeof value === "object" && value !== null && !Array.isArray(value) ? (value as Record) : null; +} + +export function asString(value: unknown): string | null { + return typeof value === "string" && value.length > 0 ? value : null; +} + +export function asInteger(value: unknown): number | null { + if (typeof value === "number") { + return Number.isSafeInteger(value) && value >= 0 ? value : null; + } + if (typeof value === "string" && /^\d+$/.test(value)) { + const parsed = Number(value); + return Number.isSafeInteger(parsed) ? parsed : null; + } + return null; +} diff --git a/apps/chain-indexer/src/akash/normalize-deployment.spec.ts b/apps/chain-indexer/src/akash/normalize-deployment.spec.ts new file mode 100644 index 0000000000..18aa2790d7 --- /dev/null +++ b/apps/chain-indexer/src/akash/normalize-deployment.spec.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from "vitest"; + +import { normalizeDeploymentMessage } from "@src/akash/normalize-deployment"; + +describe("normalizeDeploymentMessage", () => { + it("normalizes a legacy v1beta1 create with a Long dseq and a bare deposit coin", () => { + const change = normalizeDeploymentMessage("/akash.deployment.v1beta1.MsgCreateDeployment", { + id: { owner: "akash1owner", dseq: { low: 12345, high: 0, unsigned: true } }, + groups: [], + deposit: { denom: "uakt", amount: "5000000" } + }); + + expect(change).toEqual({ + kind: "deploymentCreated", + key: { owner: "akash1owner", dseq: "12345" }, + denom: "uakt", + deposit: "5000000", + depositor: null, + groups: [] + }); + }); + + it("normalizes a v1beta4 create whose deposit coin is wrapped in a Deposit message", () => { + const change = normalizeDeploymentMessage("/akash.deployment.v1beta4.MsgCreateDeployment", { + id: { owner: "akash1owner", dseq: "12345" }, + groups: [], + deposit: { amount: { denom: "ibc/170C677610AC31DF0904FFE09CD3B5C657492170E7E52372E48756B71E56F2F1", amount: "5000000" }, sources: [1] } + }) as { deposit: string; denom: string }; + + expect(change.deposit).toBe("5000000"); + expect(change.denom).toBe("ibc/170C677610AC31DF0904FFE09CD3B5C657492170E7E52372E48756B71E56F2F1"); + }); + + it("keeps the v1beta3 depositor on both create and deposit", () => { + const create = normalizeDeploymentMessage("/akash.deployment.v1beta3.MsgCreateDeployment", { + id: { owner: "akash1owner", dseq: "1" }, + groups: [], + deposit: { denom: "uakt", amount: "1" }, + depositor: "akash1other" + }) as { depositor: string }; + const deposit = normalizeDeploymentMessage("/akash.deployment.v1beta3.MsgDepositDeployment", { + id: { owner: "akash1owner", dseq: "1" }, + amount: { denom: "uakt", amount: "777" }, + depositor: "akash1other" + }); + + expect(create.depositor).toBe("akash1other"); + expect(deposit).toEqual({ + kind: "deploymentDeposited", + key: { owner: "akash1owner", dseq: "1" }, + amount: "777", + depositor: "akash1other" + }); + }); + + it("normalizes a v1 escrow deposit from its scoped xid", () => { + const change = normalizeDeploymentMessage("/akash.escrow.v1.MsgAccountDeposit", { + signer: "akash1depositor", + id: { scope: 1, xid: "akash1owner/12345" }, + deposit: { amount: { denom: "uakt", amount: "777" }, sources: [1] } + }); + + expect(change).toEqual({ + kind: "deploymentDeposited", + key: { owner: "akash1owner", dseq: "12345" }, + amount: "777", + depositor: "akash1depositor" + }); + }); + + it("normalizes a v1 escrow deposit whose scope is the string form", () => { + const change = normalizeDeploymentMessage("/akash.escrow.v1.MsgAccountDeposit", { + signer: "akash1depositor", + id: { scope: "deployment", xid: "akash1owner/12345" }, + deposit: { amount: { denom: "uakt", amount: "777" }, sources: [1] } + }); + + expect(change).toEqual({ + kind: "deploymentDeposited", + key: { owner: "akash1owner", dseq: "12345" }, + amount: "777", + depositor: "akash1depositor" + }); + }); + + it("ignores escrow deposits outside the deployment scope", () => { + const change = normalizeDeploymentMessage("/akash.escrow.v1.MsgAccountDeposit", { + signer: "akash1depositor", + id: { scope: 2, xid: "akash1owner/12345/1/1/akash1prov" }, + deposit: { amount: { denom: "uakt", amount: "777" }, sources: [1] } + }); + + expect(change).toBeNull(); + }); + + it("normalizes close, update and group lifecycle messages", () => { + const key = { owner: "akash1owner", dseq: "9" }; + + expect(normalizeDeploymentMessage("/akash.deployment.v1beta2.MsgCloseDeployment", { id: key })).toEqual({ kind: "deploymentClosed", key }); + expect(normalizeDeploymentMessage("/akash.deployment.v1beta4.MsgUpdateDeployment", { id: key })).toEqual({ kind: "deploymentUpdated", key }); + expect(normalizeDeploymentMessage("/akash.deployment.v1beta4.MsgCloseGroup", { id: { ...key, gseq: 2 } })).toEqual({ kind: "groupClosed", key, gseq: 2 }); + expect(normalizeDeploymentMessage("/akash.deployment.v1beta3.MsgPauseGroup", { id: { ...key, gseq: 1 } })).toEqual({ kind: "groupPaused", key, gseq: 1 }); + expect(normalizeDeploymentMessage("/akash.deployment.v1beta2.MsgStartGroup", { id: { ...key, gseq: 1 } })).toEqual({ kind: "groupStarted", key, gseq: 1 }); + }); + + it("returns null for unknown types and malformed bodies", () => { + expect(normalizeDeploymentMessage("/akash.deployment.v1beta1.MsgSomethingElse", {})).toBeNull(); + expect(normalizeDeploymentMessage("/akash.deployment.v1beta1.MsgCreateDeployment", { id: { owner: "" } })).toBeNull(); + expect(normalizeDeploymentMessage("/akash.deployment.v1beta1.MsgDepositDeployment", { id: { owner: "a", dseq: "1" } })).toBeNull(); + }); +}); diff --git a/apps/chain-indexer/src/akash/normalize-deployment.ts b/apps/chain-indexer/src/akash/normalize-deployment.ts new file mode 100644 index 0000000000..1e57ad4d92 --- /dev/null +++ b/apps/chain-indexer/src/akash/normalize-deployment.ts @@ -0,0 +1,115 @@ +import { type AkashChangeBody, akashTypeUrlSet, type DeploymentKey } from "@src/akash/akash-changes"; +import { asInteger, asRecord, asString } from "@src/akash/json"; +import { normalizeGroups } from "@src/akash/resources"; +import { asUint64String } from "@src/akash/uint64"; + +const DEPLOYMENT_VERSIONS = ["v1beta1", "v1beta2", "v1beta3", "v1beta4"] as const; + +const CREATE_DEPLOYMENT = typeUrlSet("MsgCreateDeployment"); +const CLOSE_DEPLOYMENT = typeUrlSet("MsgCloseDeployment"); +const UPDATE_DEPLOYMENT = typeUrlSet("MsgUpdateDeployment"); +const DEPOSIT_DEPLOYMENT = typeUrlSet("MsgDepositDeployment", ["v1beta1", "v1beta2", "v1beta3"]); +const CLOSE_GROUP = typeUrlSet("MsgCloseGroup"); +const PAUSE_GROUP = typeUrlSet("MsgPauseGroup"); +const START_GROUP = typeUrlSet("MsgStartGroup"); +const ACCOUNT_DEPOSIT = "/akash.escrow.v1.MsgAccountDeposit"; + +function typeUrlSet(name: string, versions: readonly string[] = DEPLOYMENT_VERSIONS): Set { + return akashTypeUrlSet("deployment", name, versions); +} + +export function normalizeDeploymentMessage(typeUrl: string, body: Record): AkashChangeBody | null { + if (CREATE_DEPLOYMENT.has(typeUrl)) { + return normalizeCreate(body); + } + if (CLOSE_DEPLOYMENT.has(typeUrl)) { + const key = deploymentKey(body.id); + return key ? { kind: "deploymentClosed", key } : null; + } + if (UPDATE_DEPLOYMENT.has(typeUrl)) { + const key = deploymentKey(body.id); + return key ? { kind: "deploymentUpdated", key } : null; + } + if (DEPOSIT_DEPLOYMENT.has(typeUrl)) { + return normalizeDeposit(body); + } + if (typeUrl === ACCOUNT_DEPOSIT) { + return normalizeAccountDeposit(body); + } + if (CLOSE_GROUP.has(typeUrl)) { + return normalizeGroupChange("groupClosed", body); + } + if (PAUSE_GROUP.has(typeUrl)) { + return normalizeGroupChange("groupPaused", body); + } + if (START_GROUP.has(typeUrl)) { + return normalizeGroupChange("groupStarted", body); + } + return null; +} + +function normalizeCreate(body: Record): AkashChangeBody | null { + const key = deploymentKey(body.id); + if (!key) { + return null; + } + const coin = depositCoin(body.deposit); + return { + kind: "deploymentCreated", + key, + denom: coin?.denom ?? "uakt", + deposit: coin?.amount ?? "0", + depositor: asString(body.depositor), + groups: normalizeGroups(body.groups) + }; +} + +function normalizeDeposit(body: Record): AkashChangeBody | null { + const key = deploymentKey(body.id); + const amount = asString(asRecord(body.amount)?.amount); + if (!key || !amount) { + return null; + } + return { kind: "deploymentDeposited", key, amount, depositor: asString(body.depositor) }; +} + +/** v1-era deposits target a generic escrow account: scope must be `deployment` (1) and `xid` is "owner/dseq". */ +function normalizeAccountDeposit(body: Record): AkashChangeBody | null { + const id = asRecord(body.id); + const scope = id?.scope; + if (scope !== 1 && scope !== "deployment") { + return null; + } + const [owner, dseq] = asString(id?.xid)?.split("/") ?? []; + const amount = asString(asRecord(asRecord(body.deposit)?.amount)?.amount); + if (!owner || !dseq || !amount) { + return null; + } + return { kind: "deploymentDeposited", key: { owner, dseq }, amount, depositor: asString(body.signer) }; +} + +function normalizeGroupChange(kind: "groupClosed" | "groupPaused" | "groupStarted", body: Record): AkashChangeBody | null { + const id = asRecord(body.id); + const key = deploymentKey(id); + const gseq = asInteger(id?.gseq); + return key && gseq !== null ? { kind, key, gseq } : null; +} + +/** v1beta4 wraps the deposit coin in a Deposit message (`deposit.amount`); earlier versions carry the coin directly. */ +function depositCoin(deposit: unknown): { denom: string; amount: string } | null { + const record = asRecord(deposit); + if (!record) { + return null; + } + const coin = asRecord(record.amount) ?? record; + const denom = asString(coin.denom); + const amount = asString(coin.amount); + return denom && amount ? { denom, amount } : null; +} + +export function deploymentKey(id: unknown): DeploymentKey | null { + const record = asRecord(id); + const owner = asString(record?.owner); + const dseq = asUint64String(record?.dseq); + return owner && dseq ? { owner, dseq } : null; +} diff --git a/apps/chain-indexer/src/akash/normalize-market.spec.ts b/apps/chain-indexer/src/akash/normalize-market.spec.ts new file mode 100644 index 0000000000..070ac0cdd9 --- /dev/null +++ b/apps/chain-indexer/src/akash/normalize-market.spec.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; + +import { normalizeMarketMessage } from "@src/akash/normalize-market"; + +describe("normalizeMarketMessage", () => { + it("normalizes a legacy create bid from its order id and provider field with bseq 0", () => { + const change = normalizeMarketMessage("/akash.market.v1beta2.MsgCreateBid", { + order: { owner: "akash1owner", dseq: { low: 42, high: 0, unsigned: true }, gseq: 1, oseq: 1 }, + provider: "akash1prov", + price: { denom: "uakt", amount: "50" } + }); + + expect(change).toEqual({ + kind: "bidCreated", + key: { owner: "akash1owner", dseq: "42", gseq: 1, oseq: 1, bseq: 0, provider: "akash1prov" }, + price: "50", + priceDenom: "uakt" + }); + }); + + it("normalizes a v1beta5 create bid from its full BidID with bseq and a DecCoin price", () => { + const change = normalizeMarketMessage("/akash.market.v1beta5.MsgCreateBid", { + id: { owner: "akash1owner", dseq: "42", gseq: 1, oseq: 1, bseq: 2, provider: "akash1prov" }, + price: { denom: "uakt", amount: "3.25" } + }); + + expect(change).toEqual({ + kind: "bidCreated", + key: { owner: "akash1owner", dseq: "42", gseq: 1, oseq: 1, bseq: 2, provider: "akash1prov" }, + price: "3.25", + priceDenom: "uakt" + }); + }); + + it("normalizes close bid, lease lifecycle and withdraw across id field names", () => { + const key = { owner: "akash1owner", dseq: "42", gseq: 1, oseq: 1, bseq: 0, provider: "akash1prov" }; + const legacyId = { owner: "akash1owner", dseq: "42", gseq: 1, oseq: 1, provider: "akash1prov" }; + + expect(normalizeMarketMessage("/akash.market.v1beta3.MsgCloseBid", { bidId: legacyId })).toEqual({ kind: "bidClosed", key }); + expect(normalizeMarketMessage("/akash.market.v1beta5.MsgCloseBid", { id: { ...legacyId, bseq: 1 } })).toEqual({ + kind: "bidClosed", + key: { ...key, bseq: 1 } + }); + expect(normalizeMarketMessage("/akash.market.v1beta4.MsgCreateLease", { bidId: legacyId })).toEqual({ kind: "leaseCreated", key }); + expect(normalizeMarketMessage("/akash.market.v1beta1.MsgCloseLease", { leaseId: legacyId })).toEqual({ kind: "leaseClosed", key }); + expect(normalizeMarketMessage("/akash.market.v1beta5.MsgCloseLease", { id: legacyId })).toEqual({ kind: "leaseClosed", key }); + expect(normalizeMarketMessage("/akash.market.v1beta2.MsgWithdrawLease", { bidId: legacyId })).toEqual({ kind: "leaseWithdrawn", key }); + expect(normalizeMarketMessage("/akash.market.v1beta5.MsgWithdrawLease", { id: legacyId })).toEqual({ kind: "leaseWithdrawn", key }); + }); + + it("returns null for unknown types and incomplete ids", () => { + expect(normalizeMarketMessage("/akash.market.v1beta5.MsgLeaseStartReclaim", { id: {} })).toBeNull(); + expect(normalizeMarketMessage("/akash.market.v1beta1.MsgCreateBid", { order: { owner: "a" }, provider: "p" })).toBeNull(); + }); +}); diff --git a/apps/chain-indexer/src/akash/normalize-market.ts b/apps/chain-indexer/src/akash/normalize-market.ts new file mode 100644 index 0000000000..74e5f424f6 --- /dev/null +++ b/apps/chain-indexer/src/akash/normalize-market.ts @@ -0,0 +1,65 @@ +import { type AkashChangeBody, akashTypeUrlSet, type LeaseKey } from "@src/akash/akash-changes"; +import { asInteger, asRecord, asString } from "@src/akash/json"; +import { deploymentKey } from "@src/akash/normalize-deployment"; + +const MARKET_VERSIONS = ["v1beta1", "v1beta2", "v1beta3", "v1beta4", "v1beta5"] as const; + +const CREATE_BID = typeUrlSet("MsgCreateBid"); +const CLOSE_BID = typeUrlSet("MsgCloseBid"); +const CREATE_LEASE = typeUrlSet("MsgCreateLease"); +const CLOSE_LEASE = typeUrlSet("MsgCloseLease"); +const WITHDRAW_LEASE = typeUrlSet("MsgWithdrawLease"); + +function typeUrlSet(name: string): Set { + return akashTypeUrlSet("market", name, MARKET_VERSIONS); +} + +export function normalizeMarketMessage(typeUrl: string, body: Record): AkashChangeBody | null { + if (CREATE_BID.has(typeUrl)) { + return normalizeCreateBid(body); + } + if (CLOSE_BID.has(typeUrl)) { + const key = leaseKey(body.id ?? body.bidId); + return key ? { kind: "bidClosed", key } : null; + } + if (CREATE_LEASE.has(typeUrl)) { + const key = leaseKey(body.bidId); + return key ? { kind: "leaseCreated", key } : null; + } + if (CLOSE_LEASE.has(typeUrl)) { + const key = leaseKey(body.id ?? body.leaseId); + return key ? { kind: "leaseClosed", key } : null; + } + if (WITHDRAW_LEASE.has(typeUrl)) { + const key = leaseKey(body.id ?? body.bidId); + return key ? { kind: "leaseWithdrawn", key } : null; + } + return null; +} + +/** v1beta1–4 identify the bid by OrderID + a separate provider field; v1beta5 by a full BidID with bseq. */ +function normalizeCreateBid(body: Record): AkashChangeBody | null { + const key = leaseKey(body.id) ?? leaseKey(body.order, asString(body.provider)); + if (!key) { + return null; + } + const price = asRecord(body.price); + return { + kind: "bidCreated", + key, + price: asString(price?.amount) ?? "0", + priceDenom: asString(price?.denom) ?? "" + }; +} + +function leaseKey(id: unknown, providerOverride?: string | null): LeaseKey | null { + const record = asRecord(id); + const base = deploymentKey(record); + const gseq = asInteger(record?.gseq); + const oseq = asInteger(record?.oseq); + const provider = providerOverride ?? asString(record?.provider); + if (!base || gseq === null || oseq === null || !provider) { + return null; + } + return { ...base, gseq, oseq, bseq: asInteger(record?.bseq) ?? 0, provider }; +} diff --git a/apps/chain-indexer/src/akash/resources.spec.ts b/apps/chain-indexer/src/akash/resources.spec.ts new file mode 100644 index 0000000000..56b716b105 --- /dev/null +++ b/apps/chain-indexer/src/akash/resources.spec.ts @@ -0,0 +1,131 @@ +import { describe, expect, it } from "vitest"; + +import { normalizeGroups } from "@src/akash/resources"; + +const base64Digits = (digits: string) => Buffer.from(digits, "ascii").toString("base64"); + +describe("normalizeGroups", () => { + it("normalizes v1beta1 groups with base64 values and a single storage object", () => { + const groups = normalizeGroups([ + { + name: "g1", + resources: [ + { + resources: { + cpu: { units: { val: base64Digits("1000") }, attributes: [] }, + memory: { quantity: { val: base64Digits("536870912") } }, + storage: { quantity: { val: base64Digits("268435456") }, attributes: [] } + }, + count: 2, + price: { denom: "uakt", amount: "50" } + } + ] + } + ]); + + expect(groups).toEqual([ + { + gseq: 1, + resources: [ + { + count: 2, + cpuUnits: 1000, + gpuUnits: 0, + gpuVendor: null, + gpuModel: null, + memoryBytes: 536870912, + ephemeralStorageBytes: 268435456, + persistentStorageBytes: 0, + price: "50", + priceDenom: "uakt" + } + ] + } + ]); + }); + + it("splits v1beta2+ storage arrays into ephemeral and persistent by attribute", () => { + const groups = normalizeGroups([ + { + resources: [ + { + resources: { + cpu: { units: { val: base64Digits("100") } }, + memory: { quantity: { val: base64Digits("1024") } }, + storage: [ + { quantity: { val: base64Digits("100") }, attributes: [] }, + { quantity: { val: base64Digits("200") }, attributes: [{ key: "persistent", value: "true" }] }, + { quantity: { val: base64Digits("50") }, attributes: [{ key: "persistent", value: "false" }] } + ] + }, + count: 1, + price: { denom: "uakt", amount: "1" } + } + ] + } + ]); + + expect(groups[0].resources[0].ephemeralStorageBytes).toBe(150); + expect(groups[0].resources[0].persistentStorageBytes).toBe(200); + }); + + it("normalizes chain-sdk groups with digit-string values, gpu attributes and the `resource` key", () => { + const groups = normalizeGroups([ + { + resources: [ + { + resource: { + cpu: { units: { val: "1000" } }, + gpu: { units: { val: "1" }, attributes: [{ key: "vendor/nvidia/model/a100", value: "true" }] }, + memory: { quantity: { val: "536870912" } }, + storage: [{ quantity: { val: "268435456" }, attributes: [] }] + }, + count: 3, + price: { denom: "uakt", amount: "50.5" } + } + ] + } + ]); + + expect(groups[0].resources[0]).toEqual({ + count: 3, + cpuUnits: 1000, + gpuUnits: 1, + gpuVendor: "nvidia", + gpuModel: "a100", + memoryBytes: 536870912, + ephemeralStorageBytes: 268435456, + persistentStorageBytes: 0, + price: "50.5", + priceDenom: "uakt" + }); + }); + + it("treats a wildcard gpu model as any model", () => { + const groups = normalizeGroups([ + { + resources: [ + { + resource: { gpu: { units: { val: "2" }, attributes: [{ key: "vendor/nvidia/model/*", value: "true" }] } }, + count: 1, + price: { denom: "uakt", amount: "1" } + } + ] + } + ]); + + expect(groups[0].resources[0].gpuVendor).toBe("nvidia"); + expect(groups[0].resources[0].gpuModel).toBeNull(); + }); + + it("assigns gseq by position and tolerates malformed groups", () => { + const groups = normalizeGroups([{ resources: [] }, {}, null]); + + expect(groups.map(group => group.gseq)).toEqual([1, 2, 3]); + expect(groups.every(group => group.resources.length === 0)).toBe(true); + }); + + it("returns no groups for a non-array input", () => { + expect(normalizeGroups(undefined)).toEqual([]); + }); +}); diff --git a/apps/chain-indexer/src/akash/resources.ts b/apps/chain-indexer/src/akash/resources.ts new file mode 100644 index 0000000000..b9f813c882 --- /dev/null +++ b/apps/chain-indexer/src/akash/resources.ts @@ -0,0 +1,96 @@ +import type { NormalizedGroup, NormalizedResource } from "@src/akash/akash-changes"; +import { asRecord } from "@src/akash/json"; + +/** + * Normalizes the GroupSpec list of any deployment proto era to one shape. The differences are + * structural rather than semantic, so detection is shape-driven instead of version-driven: + * v1beta1/2 nest quantities under `resources`, v1beta3+ under `resource`; v1beta1 has a single + * storage object, later versions an array; the chain SDK decodes `ResourceValue.val` to a digit + * string while the legacy package leaves a Uint8Array of ASCII digits that canonical JSON stores + * as base64. GPU exists from v1beta3 and carries vendor/model as a `vendor//model/` attribute. + */ +export function normalizeGroups(groups: unknown): NormalizedGroup[] { + if (!Array.isArray(groups)) { + return []; + } + return groups.map((group, index) => ({ + gseq: index + 1, + resources: normalizeResources(asRecord(group)?.resources) + })); +} + +function normalizeResources(units: unknown): NormalizedResource[] { + if (!Array.isArray(units)) { + return []; + } + return units.map(unit => { + const unitRecord = asRecord(unit) ?? {}; + const quantities = asRecord(unitRecord.resource) ?? asRecord(unitRecord.resources) ?? {}; + const gpu = asRecord(quantities.gpu); + const { vendor, model } = gpuAttributes(gpu); + const price = asRecord(unitRecord.price); + const storage = storageEntries(quantities.storage); + + return { + count: typeof unitRecord.count === "number" ? unitRecord.count : 0, + cpuUnits: resourceValue(asRecord(quantities.cpu)?.units), + gpuUnits: resourceValue(gpu?.units), + gpuVendor: vendor, + gpuModel: model, + memoryBytes: resourceValue(asRecord(quantities.memory)?.quantity), + ephemeralStorageBytes: sumStorage(storage, entry => !isPersistentStorage(entry)), + persistentStorageBytes: sumStorage(storage, isPersistentStorage), + price: typeof price?.amount === "string" && price.amount.length > 0 ? price.amount : "0", + priceDenom: typeof price?.denom === "string" ? price.denom : "" + }; + }); +} + +function storageEntries(storage: unknown): Array> { + if (Array.isArray(storage)) { + return storage.map(entry => asRecord(entry) ?? {}); + } + const single = asRecord(storage); + return single ? [single] : []; +} + +function sumStorage(entries: Array>, predicate: (entry: Record) => boolean): number { + return entries.filter(predicate).reduce((sum, entry) => sum + resourceValue(asRecord(entry.quantity)), 0); +} + +function isPersistentStorage(storage: Record): boolean { + return attributeList(storage.attributes).some(attribute => attribute.key === "persistent" && attribute.value === "true"); +} + +/** GPU vendor/model come from a single `vendor//model/` attribute; a `*` model means any (mirrors the legacy parser). */ +function gpuAttributes(gpu: Record | null): { vendor: string | null; model: string | null } { + const attributes = attributeList(gpu?.attributes); + if (attributes.length !== 1 || attributes[0].value !== "true") { + return { vendor: null, model: null }; + } + const match = /^vendor\/(.*)\/model\/(.*)$/.exec(attributes[0].key); + if (!match) { + return { vendor: null, model: null }; + } + return { vendor: match[1], model: match[2] !== "*" ? match[2] : null }; +} + +function attributeList(attributes: unknown): Array<{ key: string; value: string }> { + if (!Array.isArray(attributes)) { + return []; + } + return attributes.flatMap(attribute => { + const record = asRecord(attribute); + return typeof record?.key === "string" && typeof record?.value === "string" ? [{ key: record.key, value: record.value }] : []; + }); +} + +function resourceValue(container: unknown): number { + const val = asRecord(container)?.val; + if (typeof val !== "string" || val.length === 0) { + return 0; + } + const digits = /^\d+$/.test(val) ? val : Buffer.from(val, "base64").toString("ascii"); + const parsed = Number(digits); + return Number.isFinite(parsed) ? parsed : 0; +} diff --git a/apps/chain-indexer/src/akash/settlement.spec.ts b/apps/chain-indexer/src/akash/settlement.spec.ts new file mode 100644 index 0000000000..de983fd958 --- /dev/null +++ b/apps/chain-indexer/src/akash/settlement.spec.ts @@ -0,0 +1,124 @@ +import { describe, expect, it } from "vitest"; + +import { decFromInt, decFromString } from "@src/akash/dec"; +import type { SettlementDeployment, SettlementLease } from "@src/akash/settlement"; +import { settle } from "@src/akash/settlement"; + +describe("settle", () => { + it("accrues a single lease's earnings for the full height delta", () => { + const { deployment, leases } = setup({ balance: "1000000", leasePrices: ["10"], lastWithdrawHeight: 100 }); + + const result = settle(deployment, leases, 150); + + expect(result).toEqual({ blockRate: decFromInt(10), overdrawn: false }); + expect(deployment.balance).toBe(decFromInt(1000000 - 500)); + expect(deployment.lastWithdrawHeight).toBe(150); + expect(leases[0].balance).toBe(decFromInt(500)); + expect(leases[0].closedHeight).toBeNull(); + }); + + it("is a no-op when already settled at this height", () => { + const { deployment, leases } = setup({ balance: "1000", leasePrices: ["10"], lastWithdrawHeight: 150 }); + + const result = settle(deployment, leases, 150); + + expect(result).toEqual({ blockRate: decFromInt(10), overdrawn: false }); + expect(deployment.balance).toBe(decFromInt(1000)); + expect(leases[0].balance).toBe(0n); + }); + + it("only stamps the settlement height when no leases are open", () => { + const { deployment } = setup({ balance: "1000", leasePrices: [], lastWithdrawHeight: null }); + + const result = settle(deployment, [], 150); + + expect(result).toEqual({ blockRate: 0n, overdrawn: false }); + expect(deployment.lastWithdrawHeight).toBe(150); + expect(deployment.balance).toBe(decFromInt(1000)); + }); + + it("splits full-block accrual between leases at their own rates", () => { + const { deployment, leases } = setup({ balance: "10000", leasePrices: ["10", "30"], lastWithdrawHeight: 100 }); + + settle(deployment, leases, 110); + + expect(leases[0].balance).toBe(decFromInt(100)); + expect(leases[1].balance).toBe(decFromInt(300)); + expect(deployment.balance).toBe(decFromInt(10000 - 400)); + }); + + it("accrues fractional DecCoin rates exactly", () => { + const { deployment, leases } = setup({ balance: "1000", leasePrices: ["1.5"], lastWithdrawHeight: 0 }); + + settle(deployment, leases, 101); + + expect(leases[0].balance).toBe(decFromString("151.5")); + expect(deployment.balance).toBe(decFromString("848.5")); + }); + + it("distributes the remaining balance by rate weight and closes everything on overdraw", () => { + const { deployment, leases } = setup({ balance: "100", leasePrices: ["10", "30"], lastWithdrawHeight: 100 }); + + const result = settle(deployment, leases, 110); + + expect(result).toEqual({ blockRate: decFromInt(40), overdrawn: true }); + expect(leases[0].balance).toBe(decFromInt(20 + 5)); + expect(leases[1].balance).toBe(decFromInt(60 + 15)); + expect(deployment.balance).toBe(0n); + expect(deployment.closedHeight).toBe(110); + expect(deployment.lastWithdrawHeight).toBe(110); + expect(leases[0].closedHeight).toBe(110); + expect(leases[1].closedHeight).toBe(110); + }); + + it("leaves at most one unit of rounding dust on an overdraw with uneven weights", () => { + const { deployment, leases } = setup({ balance: "100", leasePrices: ["1", "1", "1"], lastWithdrawHeight: 0 }); + + const result = settle(deployment, leases, 1000); + + expect(result.overdrawn).toBe(true); + expect(deployment.balance).toBeGreaterThanOrEqual(0n); + expect(deployment.balance).toBeLessThanOrEqual(decFromInt(1)); + const totalAccrued = leases.reduce((sum, lease) => sum + lease.balance, 0n); + expect(totalAccrued + deployment.balance).toBe(decFromInt(100)); + }); + + it("accrues nothing and cannot overdraw when every open lease has a zero rate", () => { + const { deployment, leases } = setup({ balance: "1000", leasePrices: ["0"], lastWithdrawHeight: 100 }); + + const result = settle(deployment, leases, 150); + + expect(result).toEqual({ blockRate: 0n, overdrawn: false }); + expect(deployment.balance).toBe(decFromInt(1000)); + expect(deployment.lastWithdrawHeight).toBe(150); + expect(leases[0].balance).toBe(0n); + expect(deployment.closedHeight).toBeNull(); + }); + + it("matches a one-shot settlement when settled incrementally", () => { + const incremental = setup({ balance: "100000", leasePrices: ["7"], lastWithdrawHeight: 0 }); + const oneShot = setup({ balance: "100000", leasePrices: ["7"], lastWithdrawHeight: 0 }); + + settle(incremental.deployment, incremental.leases, 100); + settle(incremental.deployment, incremental.leases, 250); + settle(incremental.deployment, incremental.leases, 400); + settle(oneShot.deployment, oneShot.leases, 400); + + expect(incremental.deployment.balance).toBe(oneShot.deployment.balance); + expect(incremental.leases[0].balance).toBe(oneShot.leases[0].balance); + }); + + function setup(input: { balance: string; leasePrices: string[]; lastWithdrawHeight: number | null }) { + const deployment: SettlementDeployment = { + balance: decFromString(input.balance), + lastWithdrawHeight: input.lastWithdrawHeight, + closedHeight: null + }; + const leases: SettlementLease[] = input.leasePrices.map(price => ({ + price: decFromString(price), + balance: 0n, + closedHeight: null + })); + return { deployment, leases }; + } +}); diff --git a/apps/chain-indexer/src/akash/settlement.ts b/apps/chain-indexer/src/akash/settlement.ts new file mode 100644 index 0000000000..bd27c10bbc --- /dev/null +++ b/apps/chain-indexer/src/akash/settlement.ts @@ -0,0 +1,86 @@ +import { decMul, decMulInt, decQuo, decTruncateInt, minBigInt } from "@src/akash/dec"; + +export interface SettlementDeployment { + /** Escrow funds in Dec atomics (10^-18 of the u-denom unit). */ + balance: bigint; + lastWithdrawHeight: number | null; + closedHeight: number | null; +} + +export interface SettlementLease { + /** Per-block rate in Dec atomics; fractional since v1beta3 bids price in DecCoin. */ + price: bigint; + /** Accrued-but-unwithdrawn earnings (the on-chain payment balance); payouts truncate from here. */ + balance: bigint; + closedHeight: number | null; +} + +export interface SettlementResult { + /** Sum of the open leases' per-block rates before any overdraw close, in Dec atomics. */ + blockRate: bigint; + overdrawn: boolean; +} + +/** Escrow accounts settle to at most 1 u-denom unit of rounding dust on an overdraw close. */ +const MAX_SETTLEMENT_DUST = 10n ** 18n; + +/** The block rate is the sum of the open leases' per-block prices; callers pass the leases they consider open. */ +export function sumLeaseRate(leases: T[]): bigint { + return leases.reduce((sum, lease) => sum + lease.price, 0n); +} + +/** + * Port of akash-node x/escrow account settlement (x/escrow/keeper, accountSettle) on exact LegacyDec + * math. Mutates the passed state objects: moves the exact Dec accrual since the last settlement from + * the account funds into each open lease's unwithdrawn balance, and on overdraw distributes the + * remaining funds by rate weight and closes the deployment with all open leases. Payouts (which + * truncate to whole units, with the fraction refunded on lease close) are the caller's concern — + * settlement only accrues. + */ +export function settle(deployment: SettlementDeployment, openLeases: SettlementLease[], height: number): SettlementResult { + const blockRate = sumLeaseRate(openLeases); + + if (height === deployment.lastWithdrawHeight) return { blockRate, overdrawn: false }; + + const heightDelta = BigInt(height - (deployment.lastWithdrawHeight ?? 0)); + deployment.lastWithdrawHeight = height; + + if (openLeases.length === 0) return { blockRate: 0n, overdrawn: false }; + + if (blockRate <= 0n) return { blockRate, overdrawn: false }; + + const numFullBlocks = minBigInt(decTruncateInt(decQuo(deployment.balance, blockRate)), heightDelta); + + for (const lease of openLeases) { + lease.balance += decMulInt(lease.price, numFullBlocks); + } + deployment.balance -= decMulInt(blockRate, numFullBlocks); + + if (numFullBlocks === heightDelta) return { blockRate, overdrawn: false }; + + distributeWeighted(deployment, openLeases, blockRate, height); + return { blockRate, overdrawn: true }; +} + +function distributeWeighted(deployment: SettlementDeployment, openLeases: SettlementLease[], blockRate: bigint, height: number): void { + const remaining = deployment.balance; + let transferred = 0n; + + for (const lease of openLeases) { + const amount = decQuo(decMul(remaining, lease.price), blockRate); + lease.balance += amount; + transferred += amount; + } + + deployment.balance -= transferred; + + const dust = deployment.balance < 0n ? -deployment.balance : deployment.balance; + if (dust > MAX_SETTLEMENT_DUST) { + throw new Error(`Invalid settlement at height ${height}: ${deployment.balance} atomics remain after weighted distribution`); + } + + deployment.closedHeight = height; + for (const lease of openLeases) { + lease.closedHeight = height; + } +} diff --git a/apps/chain-indexer/src/akash/uint64.spec.ts b/apps/chain-indexer/src/akash/uint64.spec.ts new file mode 100644 index 0000000000..e7907d6977 --- /dev/null +++ b/apps/chain-indexer/src/akash/uint64.spec.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; + +import { asUint64String } from "@src/akash/uint64"; + +describe("asUint64String", () => { + it("passes through digit strings from the chain SDK codegen", () => { + expect(asUint64String("12345")).toBe("12345"); + }); + + it("accepts plain numbers", () => { + expect(asUint64String(12345)).toBe("12345"); + }); + + it("recombines legacy protobufjs Long objects", () => { + expect(asUint64String({ low: 12345, high: 0, unsigned: true })).toBe("12345"); + expect(asUint64String({ low: -1, high: 0, unsigned: true })).toBe("4294967295"); + expect(asUint64String({ low: 0, high: 1, unsigned: true })).toBe("4294967296"); + }); + + it("normalizes digit strings through BigInt, stripping leading zeros", () => { + expect(asUint64String("007")).toBe("7"); + expect(asUint64String("18446744073709551615")).toBe("18446744073709551615"); + }); + + it("rejects strings above the uint64 range", () => { + expect(asUint64String("18446744073709551616")).toBeNull(); + }); + + it("rejects unsafe, negative and fractional numbers", () => { + expect(asUint64String(2 ** 53)).toBeNull(); + expect(asUint64String(-1)).toBeNull(); + expect(asUint64String(1.5)).toBeNull(); + }); + + it("rejects Long objects whose halves are non-integer or outside 32 bits", () => { + expect(asUint64String({ low: 1.5, high: 0, unsigned: true })).toBeNull(); + expect(asUint64String({ low: 5_000_000_000, high: 0, unsigned: true })).toBeNull(); + }); + + it("rejects everything else", () => { + expect(asUint64String("12.5")).toBeNull(); + expect(asUint64String(null)).toBeNull(); + expect(asUint64String(undefined)).toBeNull(); + expect(asUint64String({})).toBeNull(); + }); +}); diff --git a/apps/chain-indexer/src/akash/uint64.ts b/apps/chain-indexer/src/akash/uint64.ts new file mode 100644 index 0000000000..4e285cee3b --- /dev/null +++ b/apps/chain-indexer/src/akash/uint64.ts @@ -0,0 +1,30 @@ +/** + * Uint64 fields reach canonical JSON in three shapes depending on the proto era: the chain SDK's + * patched codegen decodes them to bigint (serialized as a digit string), plain ts-proto uses + * number, and the frozen legacy @akashnetwork/akash-api decodes to a protobufjs Long, which + * JSON-serializes as its internal `{ low, high, unsigned }` fields. + */ +const UINT64_MAX = 2n ** 64n - 1n; + +export function asUint64String(value: unknown): string | null { + if (typeof value === "string" && /^\d+$/.test(value)) { + const parsed = BigInt(value); + return parsed <= UINT64_MAX ? parsed.toString() : null; + } + if (typeof value === "number" && Number.isSafeInteger(value) && value >= 0) { + return String(value); + } + if (isLongObject(value)) { + return ((BigInt(value.high >>> 0) << 32n) | BigInt(value.low >>> 0)).toString(); + } + return null; +} + +/** protobufjs stores each half as a signed 32-bit int, so accept the full 32-bit range; anything wider would silently truncate under `>>> 0`. */ +function isInt32Half(value: unknown): value is number { + return typeof value === "number" && Number.isInteger(value) && value >= -2_147_483_648 && value <= 4_294_967_295; +} + +function isLongObject(value: unknown): value is { low: number; high: number } { + return typeof value === "object" && value !== null && "low" in value && "high" in value && isInt32Half(value.low) && isInt32Half(value.high); +} diff --git a/apps/chain-indexer/src/db/schema.spec.ts b/apps/chain-indexer/src/db/schema.spec.ts index 562511bb2d..ef9abcd1a9 100644 --- a/apps/chain-indexer/src/db/schema.spec.ts +++ b/apps/chain-indexer/src/db/schema.spec.ts @@ -6,7 +6,13 @@ import { Accounts, AccountTxs, BalanceChanges, + Bids, Delegations, + DeploymentEvents, + DeploymentGroupResources, + DeploymentGroups, + Deployments, + Leases, MessageDeadLetters, ProposalDeposits, Proposals, @@ -120,3 +126,61 @@ describe("cosmos genesis schema", () => { expect(config.foreignKeys[0].reference().foreignColumns[0].name).toBe("id"); }); }); + +describe("akash deployment schema", () => { + it("keys deployments naturally by owner and dseq with denormalized resource totals", () => { + const config = getTableConfig(Deployments); + + expect(config.name).toBe("deployments"); + const ownerDseq = config.indexes.find(index => index.config.name === "deployments_owner_dseq_idx"); + expect(ownerDseq?.config.unique).toBe(true); + expect(config.columns.map(column => column.name)).toEqual( + expect.arrayContaining(["cpu_units", "gpu_units", "memory_bytes", "ephemeral_storage_bytes", "persistent_storage_bytes"]) + ); + }); + + it("tracks escrow state and the replay watermark on the deployment row", () => { + const config = getTableConfig(Deployments); + + expect(config.columns.map(column => column.name)).toEqual( + expect.arrayContaining(["deposit", "balance", "withdrawn_amount", "block_rate", "last_withdraw_height", "last_processed_height", "close_reason"]) + ); + }); + + it("keys groups by deployment and gseq", () => { + const config = getTableConfig(DeploymentGroups); + + const deploymentGseq = config.indexes.find(index => index.config.name === "deployment_groups_deployment_gseq_idx"); + expect(deploymentGseq?.config.unique).toBe(true); + expect(config.foreignKeys[0].reference().foreignColumns[0].name).toBe("id"); + }); + + it("keys group resources by group and position in the spec", () => { + const config = getTableConfig(DeploymentGroupResources); + + expect(config.primaryKeys[0].columns.map(column => column.name)).toEqual(["deployment_group_id", "idx"]); + }); + + it("keys bids by the full on-chain bid id and keeps them on close via state", () => { + const config = getTableConfig(Bids); + + expect(config.primaryKeys[0].columns.map(column => column.name)).toEqual(["deployment_id", "gseq", "oseq", "bseq", "provider_account_id"]); + expect(config.columns.map(column => column.name)).toContain("state"); + }); + + it("keys leases like bids and carries denormalized resource totals", () => { + const config = getTableConfig(Leases); + + expect(config.primaryKeys[0].columns.map(column => column.name)).toEqual(["deployment_id", "gseq", "oseq", "bseq", "provider_account_id"]); + expect(config.columns.map(column => column.name)).toEqual( + expect.arrayContaining(["predicted_closed_height", "withdrawn_amount", "cpu_units", "gpu_units", "memory_bytes"]) + ); + }); + + it("keys the timeline by deployment, height and ordinal so re-commits conflict instead of duplicating", () => { + const config = getTableConfig(DeploymentEvents); + + expect(config.primaryKeys[0].columns.map(column => column.name)).toEqual(["deployment_id", "height", "ordinal"]); + expect(config.columns.find(column => column.name === "tx_index")?.notNull).toBe(false); + }); +}); diff --git a/apps/chain-indexer/src/db/schema.ts b/apps/chain-indexer/src/db/schema.ts index 0ee2652e62..2290be387f 100644 --- a/apps/chain-indexer/src/db/schema.ts +++ b/apps/chain-indexer/src/db/schema.ts @@ -1,3 +1,4 @@ +import { sql } from "drizzle-orm"; import { bigint, bigserial, @@ -308,3 +309,199 @@ export const ProposalDeposits = cosmosSchema.table( }, t => [primaryKey({ columns: [t.proposalId, t.depositorAccountId, t.height] })] ); + +export const akashSchema = pgSchema("akash"); + +export const deploymentCloseReason = akashSchema.enum("deployment_close_reason", ["close_message", "overdrawn", "close_event"]); + +export const groupState = akashSchema.enum("group_state", ["open", "paused", "closed"]); + +/** `active` means the bid was accepted and became a lease; bids are kept on close (unlike the legacy indexer) so the deployment timeline can tell winning bids from losing ones. */ +export const bidState = akashSchema.enum("bid_state", ["open", "active", "closed"]); + +export const deploymentEventType = akashSchema.enum("deployment_event_type", [ + "created", + "deposited", + "updated", + "closed", + "group_closed", + "group_paused", + "group_started", + "bid_created", + "bid_closed", + "lease_created", + "lease_closed", + "lease_withdrawn" +]); + +/** + * Deployments carry denormalized resource totals (the sum over group resources of quantity × count) + * and the escrow account state, so list endpoints read one row instead of joining three levels deep. + * `balance`/`withdrawn_amount` are 18-decimal Dec values mirroring the on-chain escrow account; + * `last_withdraw_height` is the on-chain settlement checkpoint. `last_processed_height` is the + * indexer's replay watermark: blocks at or below it are duplicates and must not be re-applied, which + * — like the balance ledger — makes correctness depend on indexing a deployment's messages in height + * order from its creation (backfill from genesis, then sync). + */ +export const Deployments = akashSchema.table( + "deployments", + { + id: bigserial("id", { mode: "number" }).primaryKey(), + ownerAccountId: integer("owner_account_id") + .notNull() + .references(() => Accounts.id), + dseq: numeric("dseq", { precision: 20, scale: 0 }).notNull(), + denom: text("denom").notNull(), + deposit: numeric("deposit", { precision: 38, scale: 0 }).notNull(), + balance: numeric("balance", { precision: 38, scale: 18 }).notNull(), + withdrawnAmount: numeric("withdrawn_amount", { precision: 38, scale: 18 }).notNull(), + blockRate: numeric("block_rate", { precision: 38, scale: 18 }).notNull().default("0"), + lastWithdrawHeight: bigint("last_withdraw_height", { mode: "number" }), + lastProcessedHeight: bigint("last_processed_height", { mode: "number" }).notNull(), + createdHeight: bigint("created_height", { mode: "number" }).notNull(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull(), + closedHeight: bigint("closed_height", { mode: "number" }), + closedAt: timestamp("closed_at", { withTimezone: true }), + closeReason: deploymentCloseReason("close_reason"), + cpuUnits: bigint("cpu_units", { mode: "number" }).notNull(), + gpuUnits: bigint("gpu_units", { mode: "number" }).notNull(), + memoryBytes: bigint("memory_bytes", { mode: "number" }).notNull(), + ephemeralStorageBytes: bigint("ephemeral_storage_bytes", { mode: "number" }).notNull(), + persistentStorageBytes: bigint("persistent_storage_bytes", { mode: "number" }).notNull() + }, + t => [ + uniqueIndex("deployments_owner_dseq_idx").on(t.ownerAccountId, t.dseq), + index("deployments_owner_created_idx").on(t.ownerAccountId, t.createdHeight), + index("deployments_open_idx") + .on(t.createdHeight) + .where(sql`${t.closedHeight} IS NULL`) + ] +); + +export const DeploymentGroups = akashSchema.table( + "deployment_groups", + { + id: bigserial("id", { mode: "number" }).primaryKey(), + deploymentId: bigint("deployment_id", { mode: "number" }) + .notNull() + .references(() => Deployments.id), + gseq: integer("gseq").notNull(), + state: groupState("state").notNull().default("open"), + closedHeight: bigint("closed_height", { mode: "number" }) + }, + t => [uniqueIndex("deployment_groups_deployment_gseq_idx").on(t.deploymentId, t.gseq)] +); + +/** Immutable spec rows; `idx` is the resource's position in the on-chain GroupSpec resources array. */ +export const DeploymentGroupResources = akashSchema.table( + "deployment_group_resources", + { + deploymentGroupId: bigint("deployment_group_id", { mode: "number" }) + .notNull() + .references(() => DeploymentGroups.id), + idx: integer("idx").notNull(), + count: integer("count").notNull(), + cpuUnits: bigint("cpu_units", { mode: "number" }).notNull(), + gpuUnits: bigint("gpu_units", { mode: "number" }).notNull(), + gpuVendor: text("gpu_vendor"), + gpuModel: text("gpu_model"), + memoryBytes: bigint("memory_bytes", { mode: "number" }).notNull(), + ephemeralStorageBytes: bigint("ephemeral_storage_bytes", { mode: "number" }).notNull(), + persistentStorageBytes: bigint("persistent_storage_bytes", { mode: "number" }).notNull(), + price: numeric("price", { precision: 38, scale: 18 }).notNull(), + priceDenom: text("price_denom").notNull() + }, + t => [primaryKey({ columns: [t.deploymentGroupId, t.idx] })] +); + +export const Bids = akashSchema.table( + "bids", + { + deploymentId: bigint("deployment_id", { mode: "number" }) + .notNull() + .references(() => Deployments.id), + gseq: integer("gseq").notNull(), + oseq: integer("oseq").notNull(), + bseq: integer("bseq").notNull().default(0), + providerAccountId: integer("provider_account_id") + .notNull() + .references(() => Accounts.id), + price: numeric("price", { precision: 38, scale: 18 }).notNull(), + denom: text("denom").notNull(), + state: bidState("state").notNull().default("open"), + createdHeight: bigint("created_height", { mode: "number" }).notNull(), + closedHeight: bigint("closed_height", { mode: "number" }) + }, + t => [primaryKey({ columns: [t.deploymentId, t.gseq, t.oseq, t.bseq, t.providerAccountId] })] +); + +/** + * Leases carry the same denormalized resource totals as deployments (their group's quantity × count sums) + * so the per-block active-resource aggregation never joins group resources. `balance` is the accrued-but- + * unwithdrawn earnings mirroring the on-chain payment balance (payouts truncate to whole units and the + * fraction is refunded to the deployment on close); `predicted_closed_height` is the height at which the + * escrow balance runs out at the current block rate, recomputed on every balance- or rate-changing + * message, mirroring the legacy indexer's formulas. + */ +export const Leases = akashSchema.table( + "leases", + { + deploymentId: bigint("deployment_id", { mode: "number" }) + .notNull() + .references(() => Deployments.id), + deploymentGroupId: bigint("deployment_group_id", { mode: "number" }) + .notNull() + .references(() => DeploymentGroups.id), + gseq: integer("gseq").notNull(), + oseq: integer("oseq").notNull(), + bseq: integer("bseq").notNull().default(0), + providerAccountId: integer("provider_account_id") + .notNull() + .references(() => Accounts.id), + price: numeric("price", { precision: 38, scale: 18 }).notNull(), + denom: text("denom").notNull(), + balance: numeric("balance", { precision: 38, scale: 18 }).notNull().default("0"), + withdrawnAmount: numeric("withdrawn_amount", { precision: 38, scale: 18 }).notNull().default("0"), + predictedClosedHeight: numeric("predicted_closed_height", { precision: 30, scale: 0 }).notNull(), + createdHeight: bigint("created_height", { mode: "number" }).notNull(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull(), + closedHeight: bigint("closed_height", { mode: "number" }), + closedAt: timestamp("closed_at", { withTimezone: true }), + cpuUnits: bigint("cpu_units", { mode: "number" }).notNull(), + gpuUnits: bigint("gpu_units", { mode: "number" }).notNull(), + memoryBytes: bigint("memory_bytes", { mode: "number" }).notNull(), + ephemeralStorageBytes: bigint("ephemeral_storage_bytes", { mode: "number" }).notNull(), + persistentStorageBytes: bigint("persistent_storage_bytes", { mode: "number" }).notNull() + }, + t => [ + primaryKey({ columns: [t.deploymentId, t.gseq, t.oseq, t.bseq, t.providerAccountId] }), + index("leases_provider_idx").on(t.providerAccountId, t.closedHeight, t.createdHeight), + index("leases_open_idx") + .on(t.deploymentId) + .where(sql`${t.closedHeight} IS NULL`) + ] +); + +/** + * Typed per-deployment timeline replacing the legacy relatedMessages join. Every lifecycle change is + * stored, including withdrawals and losing bids — the legacy history view is a read-side filter, not a + * write-side decision. `ordinal` is the deterministic position of the deployment's events within the + * block, so re-committing a block conflicts instead of duplicating. `tx_index`/`msg_index` are null for + * events derived outside a message (e.g. close-event fallbacks); the tx hash comes from joining + * `cosmos.transactions`. + */ +export const DeploymentEvents = akashSchema.table( + "deployment_events", + { + deploymentId: bigint("deployment_id", { mode: "number" }) + .notNull() + .references(() => Deployments.id), + height: bigint("height", { mode: "number" }).notNull(), + ordinal: integer("ordinal").notNull(), + txIndex: integer("tx_index"), + msgIndex: integer("msg_index"), + type: deploymentEventType("type").notNull(), + details: jsonb("details") + }, + t => [primaryKey({ columns: [t.deploymentId, t.height, t.ordinal] })] +); diff --git a/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts b/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts index edf0407cbd..b1e32e3c35 100644 --- a/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts +++ b/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts @@ -3,6 +3,7 @@ import { PgDialect } from "drizzle-orm/pg-core"; import { describe, expect, it } from "vitest"; import { mock } from "vitest-mock-extended"; +import type { AkashWriter } from "@src/akash/akash-writer.service"; import { AccountTxs, Blocks, IndexerState, MessageDeadLetters, Messages, MessageTypes } from "@src/db/schema"; import type { GovWriter } from "@src/gov/gov-writer.service"; import type { AccountInterner } from "@src/pipeline/balance/account-interner.service"; @@ -236,6 +237,24 @@ describe(BlockCommitterService.name, () => { const order = insertedRows.map(call => call.table); expect(order.indexOf(AccountTxs)).toBeLessThan(order.indexOf(IndexerState)); }); + + it("hands the akash writer the derived changes and interns the addresses they reference", async () => { + const { committer, akashWriter } = setup({ selectResults: [[{ id: 7, type: MSG_SEND }]] }); + + await committer.commit( + buildBlock([MSG_SEND], 10, { + events: [{ type: "akash.deployment.v1.EventDeploymentClosed", attributes: { id: '{"owner":"akash1owner","dseq":"42"}' } }] + }) + ); + + expect(akashWriter.write).toHaveBeenCalledWith( + expect.anything(), + [expect.objectContaining({ height: 10, changes: [expect.objectContaining({ kind: "deploymentClosedEvent" })] })], + expect.any(Map) + ); + const accountIds = akashWriter.write.mock.calls[0][2]; + expect(accountIds.get("akash1owner")).toBeDefined(); + }); }); function setup(input?: { @@ -287,9 +306,10 @@ describe(BlockCommitterService.name, () => { }); const govWriter = mock(); + const akashWriter = mock(); const logger = mock(); - const committer = new BlockCommitterService(dbFake as unknown as ChainDatabase, interner, balanceWriter, govWriter, logger); - return { committer, insertedRows, conflictUpdates, deletions, interner, balanceWriter, govWriter, logger }; + const committer = new BlockCommitterService(dbFake as unknown as ChainDatabase, interner, balanceWriter, govWriter, akashWriter, logger); + return { committer, insertedRows, conflictUpdates, deletions, interner, balanceWriter, govWriter, akashWriter, logger }; } function buildBlock( diff --git a/apps/chain-indexer/src/pipeline/block-committer.service.ts b/apps/chain-indexer/src/pipeline/block-committer.service.ts index 4f438abf4e..d9568d28a8 100644 --- a/apps/chain-indexer/src/pipeline/block-committer.service.ts +++ b/apps/chain-indexer/src/pipeline/block-committer.service.ts @@ -2,6 +2,10 @@ import { and, between, inArray, isNull, sql } from "drizzle-orm"; import chunk from "lodash/chunk"; import { inject, singleton } from "tsyringe"; +import type { AkashBlockChanges } from "@src/akash/akash-changes"; +import { collectAkashAddresses } from "@src/akash/akash-changes"; +import { deriveAkashChanges } from "@src/akash/akash-deriver"; +import { AkashWriter } from "@src/akash/akash-writer.service"; import { INSERT_CHUNK_SIZE } from "@src/db/insert-chunk-size"; import { insertChunked } from "@src/db/insert-chunked"; import { AccountTxs, Blocks, IndexerState, MessageDeadLetters, Messages, MessageTypes, Transactions } from "@src/db/schema"; @@ -42,6 +46,7 @@ export class BlockCommitterService { readonly #interner: AccountInterner; readonly #balanceWriter: BalanceWriter; readonly #govWriter: GovWriter; + readonly #akashWriter: AkashWriter; readonly #logger: LoggerService; readonly #moduleRegistry = buildModuleAddressRegistry(); readonly #typeIds = new Map(); @@ -51,12 +56,14 @@ export class BlockCommitterService { @inject(AccountInterner) interner: AccountInterner, @inject(BalanceWriter) balanceWriter: BalanceWriter, @inject(GovWriter) govWriter: GovWriter, + @inject(AkashWriter) akashWriter: AkashWriter, @inject(LoggerService) logger: LoggerService ) { this.#db = db; this.#interner = interner; this.#balanceWriter = balanceWriter; this.#govWriter = govWriter; + this.#akashWriter = akashWriter; this.#logger = logger; this.#logger.setContext("COMMITTER"); } @@ -124,7 +131,8 @@ export class BlockCommitterService { const balanceChanges = blocks.flatMap(block => deriveBalanceChanges(block, this.#moduleRegistry)); const accountTxs = blocks.flatMap(block => deriveAccountTxs(block)); - const accountIds = await this.#internAccounts(balanceChanges, accountTxs); + const akashChanges = blocks.map(block => deriveAkashChanges(block)); + const accountIds = await this.#internAccounts(balanceChanges, accountTxs, akashChanges); const balanceIntents = this.#resolveBalanceChanges(balanceChanges, accountIds); const accountTxRows = this.#resolveAccountTxs(accountTxs, accountIds); @@ -139,6 +147,7 @@ export class BlockCommitterService { await this.#balanceWriter.write(tx, balanceIntents); await insertChunked(tx, AccountTxs, accountTxRows); await this.#govWriter.writeForBlocks(tx, blocks, accountIds); + await this.#akashWriter.write(tx, akashChanges, accountIds); await tx .insert(IndexerState) @@ -242,11 +251,15 @@ export class BlockCommitterService { } /** - * Interns every address the batch touches — spenders, receivers, correlated counterparties and tx signers — - * on the base connection before the commit transaction, so the ledger and activity rows can reference their - * account ids by foreign key. + * Interns every address the batch touches — spenders, receivers, correlated counterparties, tx signers, + * and the deployment owners/providers/depositors of the akash changes — on the base connection before + * the commit transaction, so the derived rows can reference their account ids by foreign key. */ - async #internAccounts(balanceChanges: DerivedBalanceChange[], accountTxs: DerivedAccountTx[]): Promise> { + async #internAccounts( + balanceChanges: DerivedBalanceChange[], + accountTxs: DerivedAccountTx[], + akashChanges: AkashBlockChanges[] + ): Promise> { const addresses = new Set(); for (const change of balanceChanges) { @@ -258,6 +271,9 @@ export class BlockCommitterService { for (const row of accountTxs) { addresses.add(row.address); } + for (const address of collectAkashAddresses(akashChanges)) { + addresses.add(address); + } return this.#interner.resolve(addresses); } diff --git a/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts b/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts index bb47b9234f..ce62247cae 100644 --- a/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts +++ b/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts @@ -1,5 +1,6 @@ import { Registry } from "@cosmjs/proto-signing"; import { defaultRegistryTypes } from "@cosmjs/stargate"; +import { MsgExec } from "cosmjs-types/cosmos/authz/v1beta1/tx"; import { MsgSend } from "cosmjs-types/cosmos/bank/v1beta1/tx"; import { AuthInfo, TxBody, TxRaw } from "cosmjs-types/cosmos/tx/v1beta1/tx"; import { createHash } from "node:crypto"; @@ -166,6 +167,59 @@ describe(BlockDecoderService.name, () => { expect(tx.events.map(e => e.type)).toEqual(["coin_spent", "coin_received"]); }); + it("captures akash close events for the deployment handler", () => { + const { decoder } = setup(); + const rawTx = buildMsgSendTx(); + const txResult: RpcTxResult = { + code: 0, + gas_used: "0", + gas_wanted: "0", + events: [ + event("akash.v1", { action: "deployment-closed", owner: "akash1owner", dseq: "3" }), + event("akash.deployment.v1.EventDeploymentClosed", { id: '{"owner":"akash1owner","dseq":"3"}' }), + event("akash.market.v1.EventLeaseClosed", { id: '{"owner":"akash1owner","dseq":"3","gseq":1,"oseq":1,"provider":"akash1prov"}' }) + ] + }; + + const [tx] = decoder.decode(buildBlock({ txs: [rawTx.toString("base64")] }), buildBlockResults([txResult])).transactions; + + expect(tx.events.map(e => e.type)).toEqual(["akash.v1", "akash.deployment.v1.EventDeploymentClosed", "akash.market.v1.EventLeaseClosed"]); + }); + + it("additively decodes MsgExec inner messages, recursing into nested execs", () => { + const { decoder } = setup(); + const innerExec = MsgExec.encode( + MsgExec.fromPartial({ grantee: "akash1inner", msgs: [{ typeUrl: "/cosmos.bank.v1beta1.MsgSend", value: encodeMsgSendValue() }] }) + ).finish(); + const outerExec = MsgExec.encode( + MsgExec.fromPartial({ grantee: "akash1outer", msgs: [{ typeUrl: "/cosmos.authz.v1beta1.MsgExec", value: innerExec }] }) + ).finish(); + const rawTx = buildMsgSendTx("/cosmos.authz.v1beta1.MsgExec", outerExec); + + const decoded = decoder.decode(buildBlock({ txs: [rawTx.toString("base64")] }), buildBlockResults([{ code: 0, gas_used: "0", gas_wanted: "0" }])); + + const body = decoded.transactions[0].messages[0].body as { + msgs: Array<{ typeUrl: string; value: string; decoded: { msgs: Array<{ decoded: unknown }> } }>; + }; + expect(body.msgs[0].typeUrl).toBe("/cosmos.authz.v1beta1.MsgExec"); + expect(body.msgs[0].value).toBe(Buffer.from(innerExec).toString("base64")); + expect(body.msgs[0].decoded.msgs[0].decoded).toMatchObject({ fromAddress: "akash1from", toAddress: "akash1to" }); + }); + + it("marks undecodable MsgExec inner messages with a null decoded field without failing the exec", () => { + const { decoder } = setup(); + const exec = MsgExec.encode( + MsgExec.fromPartial({ grantee: "akash1outer", msgs: [{ typeUrl: "/akash.unknown.v1.MsgMystery", value: new Uint8Array([1, 2, 3]) }] }) + ).finish(); + const rawTx = buildMsgSendTx("/cosmos.authz.v1beta1.MsgExec", exec); + + const decoded = decoder.decode(buildBlock({ txs: [rawTx.toString("base64")] }), buildBlockResults([{ code: 0, gas_used: "0", gas_wanted: "0" }])); + + const [message] = decoded.transactions[0].messages; + expect(message.decodeFailure).toBeUndefined(); + expect((message.body as { msgs: Array<{ decoded: unknown }> }).msgs[0].decoded).toBeNull(); + }); + it("prefers finalize_block_events for block-level events", () => { const { decoder } = setup(); diff --git a/apps/chain-indexer/src/pipeline/block-decoder.service.ts b/apps/chain-indexer/src/pipeline/block-decoder.service.ts index 16fc23c5bd..08bade8bb2 100644 --- a/apps/chain-indexer/src/pipeline/block-decoder.service.ts +++ b/apps/chain-indexer/src/pipeline/block-decoder.service.ts @@ -7,6 +7,7 @@ import type { EnvConfig } from "@src/config/env.config"; import { toCanonicalJson } from "@src/pipeline/canonical-json"; import { decodeIfBase64 } from "@src/pipeline/decode-if-base64"; import type { DecodedBlock, DecodedEvent, DecodedMessage, DecodedTransaction, MessageDecodeFailure } from "@src/pipeline/decoded-block"; +import { MAX_EXEC_DEPTH, MSG_EXEC_TYPE_URL } from "@src/pipeline/msg-exec"; import { deriveSignerAddresses } from "@src/pipeline/signer-addresses"; import { isIgnoredTypeUrl } from "@src/proto/type-catalog"; import { APP_CONFIG } from "@src/providers/app-config.provider"; @@ -16,8 +17,10 @@ import type { RpcBlockResult, RpcBlockResultsResult, RpcEvent, RpcTxResult } fro /** * The ledger derives balances and reasons from the coin/transfer/mint/burn/slash events; the gov events carry - * proposal ids and lifecycle transitions for the governance handler. Capturing the rest would waste memory - * across backfill batches. + * proposal ids and lifecycle transitions for the governance handler; the akash events catch deployment and + * lease closes that happen as side effects of other messages (group close, overdraw) — `akash.v1` is the + * legacy string-attribute event of early mainnet, the typed events are the current chain's. Capturing the + * rest would waste memory across backfill batches. */ const RELEVANT_EVENT_TYPES = new Set([ "coin_spent", @@ -28,7 +31,10 @@ const RELEVANT_EVENT_TYPES = new Set([ "slash", "submit_proposal", "active_proposal", - "inactive_proposal" + "inactive_proposal", + "akash.v1", + "akash.deployment.v1.EventDeploymentClosed", + "akash.market.v1.EventLeaseClosed" ]); const MSG_INDEX_ATTRIBUTE = "msg_index"; @@ -125,9 +131,36 @@ export class BlockDecoderService { } try { - return { body: toCanonicalJson(this.#registry.decode(message), this.#maxBodyBytes) }; + const decoded = this.#registry.decode(message); + const enriched = message.typeUrl === MSG_EXEC_TYPE_URL ? this.#decodeExecMessages(decoded, 1) : decoded; + return { body: toCanonicalJson(enriched, this.#maxBodyBytes) }; } catch (error) { return { body: null, failure: { raw: message.value, error: error instanceof Error ? error.message : String(error) } }; } } + + /** + * MsgExec carries its inner messages as Any, which canonical JSON would keep as opaque base64. Each + * inner message additionally gets a `decoded` field so handlers can read authz-wrapped messages + * (every managed-wallet deployment arrives this way). The raw `typeUrl`/`value` pair is kept, and an + * inner message that fails to decode gets `decoded: null` rather than dead-lettering the whole exec. + */ + #decodeExecMessages(decoded: unknown, depth: number): unknown { + const record = decoded as { msgs?: Array<{ typeUrl: string; value: Uint8Array }> }; + if (!Array.isArray(record.msgs)) { + return decoded; + } + + const msgs = record.msgs.map(inner => { + try { + const innerDecoded = isIgnoredTypeUrl(inner.typeUrl) ? null : this.#registry.decode(inner); + const enriched = inner.typeUrl === MSG_EXEC_TYPE_URL && depth < MAX_EXEC_DEPTH ? this.#decodeExecMessages(innerDecoded, depth + 1) : innerDecoded; + return { ...inner, decoded: enriched }; + } catch { + return { ...inner, decoded: null }; + } + }); + + return { ...record, msgs }; + } } diff --git a/apps/chain-indexer/src/pipeline/msg-exec.ts b/apps/chain-indexer/src/pipeline/msg-exec.ts new file mode 100644 index 0000000000..8cef42f99d --- /dev/null +++ b/apps/chain-indexer/src/pipeline/msg-exec.ts @@ -0,0 +1,5 @@ +/** authz MsgExec type URL; managed-wallet deployments arrive wrapped in one (occasionally two) of these. */ +export const MSG_EXEC_TYPE_URL = "/cosmos.authz.v1beta1.MsgExec"; + +/** MsgExec nested in MsgExec is legal on-chain; two levels covers every observed use without unbounded recursion. The decoder enriches and the deriver walks to the same depth so the two passes stay in step. */ +export const MAX_EXEC_DEPTH = 2; diff --git a/apps/chain-indexer/src/proto/legacy-dec-coin-precision.spec.ts b/apps/chain-indexer/src/proto/legacy-dec-coin-precision.spec.ts new file mode 100644 index 0000000000..4d6a7d659b --- /dev/null +++ b/apps/chain-indexer/src/proto/legacy-dec-coin-precision.spec.ts @@ -0,0 +1,28 @@ +import "./legacy-dec-coin-precision"; + +import { MsgCreateBid } from "@akashnetwork/akash-api/akash/market/v1beta2"; +import { DecCoin } from "@akashnetwork/akash-api/cosmos/base/v1beta1"; +import { describe, expect, it } from "vitest"; + +describe("legacy DecCoin precision", () => { + it.each([ + ["117.73952", "117.73952"], + ["100", "100"], + ["0.5", "0.5"], + ["0", "0"], + ["1000000", "1000000"], + ["0.000000000000000001", "0.000000000000000001"], + ["123456789.123456789123456789", "123456789.123456789123456789"] + ])("round-trips %s exactly instead of float-approximating it", (amount, expected) => { + const bytes = DecCoin.encode({ $type: DecCoin.$type, denom: "uakt", amount }).finish(); + + expect(DecCoin.decode(bytes)).toEqual({ $type: DecCoin.$type, denom: "uakt", amount: expected }); + }); + + it("decodes DecCoins nested inside legacy messages exactly", () => { + const message = MsgCreateBid.fromPartial({ price: { denom: "uakt", amount: "117.73952" } }); + const bytes = MsgCreateBid.encode(message).finish(); + + expect(MsgCreateBid.decode(bytes).price?.amount).toBe("117.73952"); + }); +}); diff --git a/apps/chain-indexer/src/proto/legacy-dec-coin-precision.ts b/apps/chain-indexer/src/proto/legacy-dec-coin-precision.ts new file mode 100644 index 0000000000..4112a405d1 --- /dev/null +++ b/apps/chain-indexer/src/proto/legacy-dec-coin-precision.ts @@ -0,0 +1,50 @@ +import { DecCoin } from "@akashnetwork/akash-api/cosmos/base/v1beta1"; +import { Reader } from "protobufjs/minimal"; + +/** + * `@akashnetwork/akash-api` patches `DecCoin.decode` to convert the wire-format 1e18-scaled + * atomics string into a human decimal via `parseInt(amount) / 1e18` — float64 math that corrupts + * every legacy-era DecCoin at the ~15th significant digit (a v1beta2 bid price of `117.73952` + * decodes as `117.739519999999999`). All legacy proto versions share this one module instance, + * so replacing its `decode` with exact string math fixes v1beta1–v1beta4 decoding in one place. + * Imported for its side effect by the type catalog before any registry decoding happens. + */ +export function installExactLegacyDecCoinDecode(): void { + DecCoin.decode = decodeDecCoinExact; +} + +const DENOM_FIELD = 1; +const AMOUNT_FIELD = 2; + +function decodeDecCoinExact(input: Reader | Uint8Array, length?: number): DecCoin { + const reader = input instanceof Uint8Array ? Reader.create(input) : input; + const end = length === undefined ? reader.len : reader.pos + length; + const message: DecCoin = { $type: DecCoin.$type, denom: "", amount: "" }; + + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case DENOM_FIELD: + message.denom = reader.string(); + break; + case AMOUNT_FIELD: + message.amount = decimalStringFromAtomics(reader.string()); + break; + default: + reader.skipType(tag & 7); + break; + } + } + + return message; +} + +function decimalStringFromAtomics(atomics: string): string { + const negative = atomics.startsWith("-"); + const digits = (negative ? atomics.slice(1) : atomics).padStart(19, "0"); + const whole = digits.slice(0, -18); + const fraction = digits.slice(-18).replace(/0+$/, ""); + return `${negative ? "-" : ""}${whole}${fraction ? `.${fraction}` : ""}`; +} + +installExactLegacyDecCoinDecode(); diff --git a/apps/chain-indexer/src/proto/type-catalog.ts b/apps/chain-indexer/src/proto/type-catalog.ts index 0f98d88da4..711692e4ac 100644 --- a/apps/chain-indexer/src/proto/type-catalog.ts +++ b/apps/chain-indexer/src/proto/type-catalog.ts @@ -1,3 +1,5 @@ +import "./legacy-dec-coin-precision"; + import * as legacyV1beta1 from "@akashnetwork/akash-api/v1beta1"; import * as legacyV1beta2 from "@akashnetwork/akash-api/v1beta2"; import * as legacyV1beta3 from "@akashnetwork/akash-api/v1beta3"; diff --git a/package-lock.json b/package-lock.json index ce10e800f9..65acb31937 100644 --- a/package-lock.json +++ b/package-lock.json @@ -556,6 +556,7 @@ "http-errors": "^2.0.0", "lodash": "^4.17.21", "postgres": "^3.4.4", + "protobufjs": "~6.11.2", "reflect-metadata": "^0.2.2", "tsyringe": "^4.10.0", "undici": "^7.22.0", @@ -21711,7 +21712,6 @@ "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.2.0.tgz", "integrity": "sha512-8yQrvS6sMpSwIovhPOwfyNf2Wz6v/B62LFSVYQ85+Rq3tLsBIG7rP5geMxaijTUxSkrO6RzN/IRuIAADYQsleA==", "license": "MIT", - "peer": true, "dependencies": { "@types/react": "*" } From 6a5066f1bf9795b03e44a0686a824ec765fe4931 Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Mon, 17 Aug 2026 11:58:37 +0530 Subject: [PATCH 10/14] feat(indexer): akash provider and audit attribute handlers (#3602) * feat(indexer): akash provider and audit attribute handlers Handles MsgCreate/Update/DeleteProvider across provider v1beta1-v1beta4 and MsgSign/DeleteProviderAttributes across audit v1beta1-v1beta3 + v1, writing to typed akash.providers and akash.provider_audit_signatures tables via a new ProviderWriter with watermark-guarded upserts. Part of CON-812 * refactor(indexer): dedupe ProviderAttribute type and tidy provider change guards Code-review cleanups on the provider/audit handlers, no behavior change: - single ProviderAttribute definition in akash-changes, imported by schema/resources - guards isProviderRegistryChange/isProviderAuditChange replace ad-hoc kind cascades - rename normalizeUpsert -> normalizeProviderInfo (names behavior, not the DB op) * fix(indexer): serialize provider audit-signature writes to avoid deadlock Overlapping committers (sync pods during a rolling deploy, or sync racing a backfill) apply audit signs and deletes in block order, so two writers with different block windows can lock the same audit rows in opposite orders and deadlock. The provider path already guards this with an ordered FOR UPDATE, but the audit path did not. Take one transaction-scoped advisory lock over the audit section before writing, reusing the mechanism already used for the migration lock. Audit traffic is sparse, so the lock is near-free in steady state. Also assert the provider state load locks rows FOR UPDATE in owner-account order, which no test previously covered. --- .../drizzle/0007_secret_overlord.sql | 24 + .../drizzle/meta/0007_snapshot.json | 2321 +++++++++++++++++ apps/chain-indexer/drizzle/meta/_journal.json | 7 + .../src/akash/akash-changes.spec.ts | 46 + apps/chain-indexer/src/akash/akash-changes.ts | 41 + .../src/akash/akash-deriver.spec.ts | 70 + apps/chain-indexer/src/akash/akash-deriver.ts | 10 +- .../src/akash/akash-writer.service.spec.ts | 14 + .../src/akash/akash-writer.service.ts | 7 + .../src/akash/deployment-reducer.spec.ts | 9 + .../src/akash/deployment-reducer.ts | 4 + .../src/akash/normalize-audit.spec.ts | 62 + .../src/akash/normalize-audit.ts | 35 + .../src/akash/normalize-provider.spec.ts | 74 + .../src/akash/normalize-provider.ts | 46 + .../src/akash/provider-writer.service.spec.ts | 309 +++ .../src/akash/provider-writer.service.ts | 280 ++ apps/chain-indexer/src/akash/resources.ts | 4 +- apps/chain-indexer/src/db/schema.spec.ts | 24 + apps/chain-indexer/src/db/schema.ts | 46 + .../pipeline/block-committer.service.spec.ts | 18 +- .../src/pipeline/block-committer.service.ts | 5 + 22 files changed, 3450 insertions(+), 6 deletions(-) create mode 100644 apps/chain-indexer/drizzle/0007_secret_overlord.sql create mode 100644 apps/chain-indexer/drizzle/meta/0007_snapshot.json create mode 100644 apps/chain-indexer/src/akash/akash-changes.spec.ts create mode 100644 apps/chain-indexer/src/akash/normalize-audit.spec.ts create mode 100644 apps/chain-indexer/src/akash/normalize-audit.ts create mode 100644 apps/chain-indexer/src/akash/normalize-provider.spec.ts create mode 100644 apps/chain-indexer/src/akash/normalize-provider.ts create mode 100644 apps/chain-indexer/src/akash/provider-writer.service.spec.ts create mode 100644 apps/chain-indexer/src/akash/provider-writer.service.ts diff --git a/apps/chain-indexer/drizzle/0007_secret_overlord.sql b/apps/chain-indexer/drizzle/0007_secret_overlord.sql new file mode 100644 index 0000000000..f84d5497bc --- /dev/null +++ b/apps/chain-indexer/drizzle/0007_secret_overlord.sql @@ -0,0 +1,24 @@ +CREATE TABLE "akash"."provider_audit_signatures" ( + "owner_account_id" integer NOT NULL, + "auditor_account_id" integer NOT NULL, + "key" text NOT NULL, + "value" text NOT NULL, + "height" bigint NOT NULL, + CONSTRAINT "provider_audit_signatures_owner_account_id_auditor_account_id_key_pk" PRIMARY KEY("owner_account_id","auditor_account_id","key") +); +--> statement-breakpoint +CREATE TABLE "akash"."providers" ( + "owner_account_id" integer PRIMARY KEY NOT NULL, + "host_uri" text NOT NULL, + "email" text, + "website" text, + "attributes" jsonb NOT NULL, + "last_processed_height" bigint NOT NULL, + "created_height" bigint NOT NULL, + "updated_height" bigint, + "deleted_height" bigint +); +--> statement-breakpoint +ALTER TABLE "akash"."provider_audit_signatures" ADD CONSTRAINT "provider_audit_signatures_owner_account_id_accounts_id_fk" FOREIGN KEY ("owner_account_id") REFERENCES "cosmos"."accounts"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "akash"."provider_audit_signatures" ADD CONSTRAINT "provider_audit_signatures_auditor_account_id_accounts_id_fk" FOREIGN KEY ("auditor_account_id") REFERENCES "cosmos"."accounts"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "akash"."providers" ADD CONSTRAINT "providers_owner_account_id_accounts_id_fk" FOREIGN KEY ("owner_account_id") REFERENCES "cosmos"."accounts"("id") ON DELETE no action ON UPDATE no action; \ No newline at end of file diff --git a/apps/chain-indexer/drizzle/meta/0007_snapshot.json b/apps/chain-indexer/drizzle/meta/0007_snapshot.json new file mode 100644 index 0000000000..cc6c10eb4e --- /dev/null +++ b/apps/chain-indexer/drizzle/meta/0007_snapshot.json @@ -0,0 +1,2321 @@ +{ + "id": "a2860b14-74a0-48d3-825f-dc6cecbee0d3", + "prevId": "7703e3c3-9988-452e-ae30-44e28b863168", + "version": "7", + "dialect": "postgresql", + "tables": { + "cosmos.account_balances": { + "name": "account_balances", + "schema": "cosmos", + "columns": { + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "amount": { + "name": "amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "account_balances_account_id_accounts_id_fk": { + "name": "account_balances_account_id_accounts_id_fk", + "tableFrom": "account_balances", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_balances_account_id_denom_pk": { + "name": "account_balances_account_id_denom_pk", + "columns": [ + "account_id", + "denom" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.account_txs": { + "name": "account_txs", + "schema": "cosmos", + "columns": { + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "account_tx_role", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "account_txs_account_id_accounts_id_fk": { + "name": "account_txs_account_id_accounts_id_fk", + "tableFrom": "account_txs", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_txs_account_id_height_tx_index_role_pk": { + "name": "account_txs_account_id_height_tx_index_role_pk", + "columns": [ + "account_id", + "height", + "tx_index", + "role" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.accounts": { + "name": "accounts", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_number": { + "name": "account_number", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "account_type": { + "name": "account_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_module_account": { + "name": "is_module_account", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": { + "accounts_address_idx": { + "name": "accounts_address_idx", + "columns": [ + { + "expression": "address", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.balance_changes": { + "name": "balance_changes", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delta": { + "name": "delta", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "balance_after": { + "name": "balance_after", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "balance_change_reason", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "event_index": { + "name": "event_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "counterparty_account_id": { + "name": "counterparty_account_id", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "balance_changes_account_denom_height_idx": { + "name": "balance_changes_account_denom_height_idx", + "columns": [ + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "denom", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "balance_changes_height_event_index_idx": { + "name": "balance_changes_height_event_index_idx", + "columns": [ + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_index", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "balance_changes_account_id_accounts_id_fk": { + "name": "balance_changes_account_id_accounts_id_fk", + "tableFrom": "balance_changes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "balance_changes_counterparty_account_id_accounts_id_fk": { + "name": "balance_changes_counterparty_account_id_accounts_id_fk", + "tableFrom": "balance_changes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "counterparty_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.bids": { + "name": "bids", + "schema": "akash", + "columns": { + "deployment_id": { + "name": "deployment_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gseq": { + "name": "gseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "oseq": { + "name": "oseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "bseq": { + "name": "bseq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "provider_account_id": { + "name": "provider_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "price": { + "name": "price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "bid_state", + "typeSchema": "akash", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "created_height": { + "name": "created_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "closed_height": { + "name": "closed_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "bids_deployment_id_deployments_id_fk": { + "name": "bids_deployment_id_deployments_id_fk", + "tableFrom": "bids", + "tableTo": "deployments", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "bids_provider_account_id_accounts_id_fk": { + "name": "bids_provider_account_id_accounts_id_fk", + "tableFrom": "bids", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "provider_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "bids_deployment_id_gseq_oseq_bseq_provider_account_id_pk": { + "name": "bids_deployment_id_gseq_oseq_bseq_provider_account_id_pk", + "columns": [ + "deployment_id", + "gseq", + "oseq", + "bseq", + "provider_account_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.blocks": { + "name": "blocks", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "datetime": { + "name": "datetime", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "parent_hash": { + "name": "parent_hash", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "proposer_address": { + "name": "proposer_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tx_count": { + "name": "tx_count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.delegations": { + "name": "delegations", + "schema": "cosmos", + "columns": { + "delegator_account_id": { + "name": "delegator_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "validator_operator_address": { + "name": "validator_operator_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "shares": { + "name": "shares", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "delegations_delegator_account_id_accounts_id_fk": { + "name": "delegations_delegator_account_id_accounts_id_fk", + "tableFrom": "delegations", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "delegator_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "delegations_delegator_account_id_validator_operator_address_pk": { + "name": "delegations_delegator_account_id_validator_operator_address_pk", + "columns": [ + "delegator_account_id", + "validator_operator_address" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.deployment_events": { + "name": "deployment_events", + "schema": "akash", + "columns": { + "deployment_id": { + "name": "deployment_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ordinal": { + "name": "ordinal", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "msg_index": { + "name": "msg_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "deployment_event_type", + "typeSchema": "akash", + "primaryKey": false, + "notNull": true + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "deployment_events_deployment_id_deployments_id_fk": { + "name": "deployment_events_deployment_id_deployments_id_fk", + "tableFrom": "deployment_events", + "tableTo": "deployments", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "deployment_events_deployment_id_height_ordinal_pk": { + "name": "deployment_events_deployment_id_height_ordinal_pk", + "columns": [ + "deployment_id", + "height", + "ordinal" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.deployment_group_resources": { + "name": "deployment_group_resources", + "schema": "akash", + "columns": { + "deployment_group_id": { + "name": "deployment_group_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "idx": { + "name": "idx", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "cpu_units": { + "name": "cpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gpu_units": { + "name": "gpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gpu_vendor": { + "name": "gpu_vendor", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "gpu_model": { + "name": "gpu_model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "memory_bytes": { + "name": "memory_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ephemeral_storage_bytes": { + "name": "ephemeral_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "persistent_storage_bytes": { + "name": "persistent_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "price": { + "name": "price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "price_denom": { + "name": "price_denom", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "deployment_group_resources_deployment_group_id_deployment_groups_id_fk": { + "name": "deployment_group_resources_deployment_group_id_deployment_groups_id_fk", + "tableFrom": "deployment_group_resources", + "tableTo": "deployment_groups", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_group_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "deployment_group_resources_deployment_group_id_idx_pk": { + "name": "deployment_group_resources_deployment_group_id_idx_pk", + "columns": [ + "deployment_group_id", + "idx" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.deployment_groups": { + "name": "deployment_groups", + "schema": "akash", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "deployment_id": { + "name": "deployment_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gseq": { + "name": "gseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "group_state", + "typeSchema": "akash", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "closed_height": { + "name": "closed_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "deployment_groups_deployment_gseq_idx": { + "name": "deployment_groups_deployment_gseq_idx", + "columns": [ + { + "expression": "deployment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "gseq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "deployment_groups_deployment_id_deployments_id_fk": { + "name": "deployment_groups_deployment_id_deployments_id_fk", + "tableFrom": "deployment_groups", + "tableTo": "deployments", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.deployments": { + "name": "deployments", + "schema": "akash", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "owner_account_id": { + "name": "owner_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "dseq": { + "name": "dseq", + "type": "numeric(20, 0)", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deposit": { + "name": "deposit", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "balance": { + "name": "balance", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "withdrawn_amount": { + "name": "withdrawn_amount", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "block_rate": { + "name": "block_rate", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "last_withdraw_height": { + "name": "last_withdraw_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "last_processed_height": { + "name": "last_processed_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_height": { + "name": "created_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "closed_height": { + "name": "closed_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "closed_at": { + "name": "closed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "close_reason": { + "name": "close_reason", + "type": "deployment_close_reason", + "typeSchema": "akash", + "primaryKey": false, + "notNull": false + }, + "cpu_units": { + "name": "cpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gpu_units": { + "name": "gpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "memory_bytes": { + "name": "memory_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ephemeral_storage_bytes": { + "name": "ephemeral_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "persistent_storage_bytes": { + "name": "persistent_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "deployments_owner_dseq_idx": { + "name": "deployments_owner_dseq_idx", + "columns": [ + { + "expression": "owner_account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dseq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "deployments_owner_created_idx": { + "name": "deployments_owner_created_idx", + "columns": [ + { + "expression": "owner_account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "deployments_open_idx": { + "name": "deployments_open_idx", + "columns": [ + { + "expression": "created_height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"akash\".\"deployments\".\"closed_height\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "deployments_owner_account_id_accounts_id_fk": { + "name": "deployments_owner_account_id_accounts_id_fk", + "tableFrom": "deployments", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "owner_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.indexer_state": { + "name": "indexer_state", + "schema": "", + "columns": { + "stream": { + "name": "stream", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "last_height": { + "name": "last_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.leases": { + "name": "leases", + "schema": "akash", + "columns": { + "deployment_id": { + "name": "deployment_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "deployment_group_id": { + "name": "deployment_group_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gseq": { + "name": "gseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "oseq": { + "name": "oseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "bseq": { + "name": "bseq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "provider_account_id": { + "name": "provider_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "price": { + "name": "price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "balance": { + "name": "balance", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "withdrawn_amount": { + "name": "withdrawn_amount", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "predicted_closed_height": { + "name": "predicted_closed_height", + "type": "numeric(30, 0)", + "primaryKey": false, + "notNull": true + }, + "created_height": { + "name": "created_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "closed_height": { + "name": "closed_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "closed_at": { + "name": "closed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "cpu_units": { + "name": "cpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gpu_units": { + "name": "gpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "memory_bytes": { + "name": "memory_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ephemeral_storage_bytes": { + "name": "ephemeral_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "persistent_storage_bytes": { + "name": "persistent_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "leases_provider_idx": { + "name": "leases_provider_idx", + "columns": [ + { + "expression": "provider_account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "closed_height", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "leases_open_idx": { + "name": "leases_open_idx", + "columns": [ + { + "expression": "deployment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"akash\".\"leases\".\"closed_height\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "leases_deployment_id_deployments_id_fk": { + "name": "leases_deployment_id_deployments_id_fk", + "tableFrom": "leases", + "tableTo": "deployments", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "leases_deployment_group_id_deployment_groups_id_fk": { + "name": "leases_deployment_group_id_deployment_groups_id_fk", + "tableFrom": "leases", + "tableTo": "deployment_groups", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_group_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "leases_provider_account_id_accounts_id_fk": { + "name": "leases_provider_account_id_accounts_id_fk", + "tableFrom": "leases", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "provider_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "leases_deployment_id_gseq_oseq_bseq_provider_account_id_pk": { + "name": "leases_deployment_id_gseq_oseq_bseq_provider_account_id_pk", + "columns": [ + "deployment_id", + "gseq", + "oseq", + "bseq", + "provider_account_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.message_dead_letters": { + "name": "message_dead_letters", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type_id": { + "name": "type_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "raw": { + "name": "raw", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "message_dead_letters_type_id_idx": { + "name": "message_dead_letters_type_id_idx", + "columns": [ + { + "expression": "type_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "message_dead_letters_type_id_message_types_id_fk": { + "name": "message_dead_letters_type_id_message_types_id_fk", + "tableFrom": "message_dead_letters", + "tableTo": "message_types", + "schemaTo": "cosmos", + "columnsFrom": [ + "type_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "message_dead_letters_height_tx_index_index_pk": { + "name": "message_dead_letters_height_tx_index_index_pk", + "columns": [ + "height", + "tx_index", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.message_types": { + "name": "message_types", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "message_types_type_idx": { + "name": "message_types_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.messages": { + "name": "messages", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type_id": { + "name": "type_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "body": { + "name": "body", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "messages_type_id_idx": { + "name": "messages_type_id_idx", + "columns": [ + { + "expression": "type_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "messages_type_id_message_types_id_fk": { + "name": "messages_type_id_message_types_id_fk", + "tableFrom": "messages", + "tableTo": "message_types", + "schemaTo": "cosmos", + "columnsFrom": [ + "type_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messages_height_tx_index_index_pk": { + "name": "messages_height_tx_index_index_pk", + "columns": [ + "height", + "tx_index", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.proposal_deposits": { + "name": "proposal_deposits", + "schema": "cosmos", + "columns": { + "proposal_id": { + "name": "proposal_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "depositor_account_id": { + "name": "depositor_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "amount": { + "name": "amount", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "proposal_deposits_depositor_account_id_accounts_id_fk": { + "name": "proposal_deposits_depositor_account_id_accounts_id_fk", + "tableFrom": "proposal_deposits", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "depositor_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "proposal_deposits_proposal_id_depositor_account_id_height_pk": { + "name": "proposal_deposits_proposal_id_depositor_account_id_height_pk", + "columns": [ + "proposal_id", + "depositor_account_id", + "height" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.proposal_votes": { + "name": "proposal_votes", + "schema": "cosmos", + "columns": { + "proposal_id": { + "name": "proposal_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "voter_account_id": { + "name": "voter_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "options": { + "name": "options", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "proposal_votes_voter_account_id_accounts_id_fk": { + "name": "proposal_votes_voter_account_id_accounts_id_fk", + "tableFrom": "proposal_votes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "voter_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "proposal_votes_proposal_id_voter_account_id_pk": { + "name": "proposal_votes_proposal_id_voter_account_id_pk", + "columns": [ + "proposal_id", + "voter_account_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.proposals": { + "name": "proposals", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "proposer_account_id": { + "name": "proposer_account_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "messages": { + "name": "messages", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "proposal_status", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + }, + "submit_time": { + "name": "submit_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deposit_end_time": { + "name": "deposit_end_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "voting_start_time": { + "name": "voting_start_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "voting_end_time": { + "name": "voting_end_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "total_deposit": { + "name": "total_deposit", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "final_tally_yes": { + "name": "final_tally_yes", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "final_tally_abstain": { + "name": "final_tally_abstain", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "final_tally_no": { + "name": "final_tally_no", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "final_tally_no_with_veto": { + "name": "final_tally_no_with_veto", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "submit_height": { + "name": "submit_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "proposals_proposer_account_id_accounts_id_fk": { + "name": "proposals_proposer_account_id_accounts_id_fk", + "tableFrom": "proposals", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "proposer_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.provider_audit_signatures": { + "name": "provider_audit_signatures", + "schema": "akash", + "columns": { + "owner_account_id": { + "name": "owner_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "auditor_account_id": { + "name": "auditor_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "provider_audit_signatures_owner_account_id_accounts_id_fk": { + "name": "provider_audit_signatures_owner_account_id_accounts_id_fk", + "tableFrom": "provider_audit_signatures", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "owner_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "provider_audit_signatures_auditor_account_id_accounts_id_fk": { + "name": "provider_audit_signatures_auditor_account_id_accounts_id_fk", + "tableFrom": "provider_audit_signatures", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "auditor_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "provider_audit_signatures_owner_account_id_auditor_account_id_key_pk": { + "name": "provider_audit_signatures_owner_account_id_auditor_account_id_key_pk", + "columns": [ + "owner_account_id", + "auditor_account_id", + "key" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.providers": { + "name": "providers", + "schema": "akash", + "columns": { + "owner_account_id": { + "name": "owner_account_id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "host_uri": { + "name": "host_uri", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attributes": { + "name": "attributes", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "last_processed_height": { + "name": "last_processed_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_height": { + "name": "created_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "updated_height": { + "name": "updated_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "deleted_height": { + "name": "deleted_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "providers_owner_account_id_accounts_id_fk": { + "name": "providers_owner_account_id_accounts_id_fk", + "tableFrom": "providers", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "owner_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.transactions": { + "name": "transactions", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "code": { + "name": "code", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "gas_used": { + "name": "gas_used", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gas_wanted": { + "name": "gas_wanted", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "fee": { + "name": "fee", + "type": "jsonb", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "transactions_hash_idx": { + "name": "transactions_hash_idx", + "columns": [ + { + "expression": "hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "transactions_height_index_pk": { + "name": "transactions_height_index_pk", + "columns": [ + "height", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.unbonding_delegations": { + "name": "unbonding_delegations", + "schema": "cosmos", + "columns": { + "delegator_account_id": { + "name": "delegator_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "validator_operator_address": { + "name": "validator_operator_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "creation_height": { + "name": "creation_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "completion_time": { + "name": "completion_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "initial_balance": { + "name": "initial_balance", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "balance": { + "name": "balance", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "unbonding_delegations_delegator_account_id_accounts_id_fk": { + "name": "unbonding_delegations_delegator_account_id_accounts_id_fk", + "tableFrom": "unbonding_delegations", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "delegator_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "unbonding_delegations_delegator_account_id_validator_operator_address_creation_height_pk": { + "name": "unbonding_delegations_delegator_account_id_validator_operator_address_creation_height_pk", + "columns": [ + "delegator_account_id", + "validator_operator_address", + "creation_height" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.validators": { + "name": "validators", + "schema": "cosmos", + "columns": { + "operator_address": { + "name": "operator_address", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_address": { + "name": "account_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "hex_address": { + "name": "hex_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "moniker": { + "name": "moniker", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "identity": { + "name": "identity", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "details": { + "name": "details", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "security_contact": { + "name": "security_contact", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commission_rate": { + "name": "commission_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "commission_max_rate": { + "name": "commission_max_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "commission_max_change_rate": { + "name": "commission_max_change_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "min_self_delegation": { + "name": "min_self_delegation", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "jailed": { + "name": "jailed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "status": { + "name": "status", + "type": "validator_status", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": false + }, + "tokens": { + "name": "tokens", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "delegator_shares": { + "name": "delegator_shares", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": false + }, + "unbonding_height": { + "name": "unbonding_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "unbonding_time": { + "name": "unbonding_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "cosmos.account_tx_role": { + "name": "account_tx_role", + "schema": "cosmos", + "values": [ + "signer", + "sender", + "receiver" + ] + }, + "cosmos.balance_change_reason": { + "name": "balance_change_reason", + "schema": "cosmos", + "values": [ + "genesis", + "transfer", + "fee", + "reward", + "commission", + "slash", + "gov", + "ibc", + "escrow", + "bme", + "mint", + "burn", + "staking" + ] + }, + "akash.bid_state": { + "name": "bid_state", + "schema": "akash", + "values": [ + "open", + "active", + "closed" + ] + }, + "akash.deployment_close_reason": { + "name": "deployment_close_reason", + "schema": "akash", + "values": [ + "close_message", + "overdrawn", + "close_event" + ] + }, + "akash.deployment_event_type": { + "name": "deployment_event_type", + "schema": "akash", + "values": [ + "created", + "deposited", + "updated", + "closed", + "group_closed", + "group_paused", + "group_started", + "bid_created", + "bid_closed", + "lease_created", + "lease_closed", + "lease_withdrawn" + ] + }, + "akash.group_state": { + "name": "group_state", + "schema": "akash", + "values": [ + "open", + "paused", + "closed" + ] + }, + "cosmos.proposal_status": { + "name": "proposal_status", + "schema": "cosmos", + "values": [ + "deposit_period", + "voting_period", + "passed", + "rejected", + "failed" + ] + }, + "cosmos.validator_status": { + "name": "validator_status", + "schema": "cosmos", + "values": [ + "unbonded", + "unbonding", + "bonded" + ] + }, + "cosmos.vote_option": { + "name": "vote_option", + "schema": "cosmos", + "values": [ + "yes", + "abstain", + "no", + "no_with_veto" + ] + } + }, + "schemas": { + "akash": "akash", + "cosmos": "cosmos" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/chain-indexer/drizzle/meta/_journal.json b/apps/chain-indexer/drizzle/meta/_journal.json index eda4b50e95..dc53cc0966 100644 --- a/apps/chain-indexer/drizzle/meta/_journal.json +++ b/apps/chain-indexer/drizzle/meta/_journal.json @@ -50,6 +50,13 @@ "when": 1786877216729, "tag": "0006_next_earthquake", "breakpoints": true + }, + { + "idx": 7, + "version": "7", + "when": 1786898959325, + "tag": "0007_secret_overlord", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/chain-indexer/src/akash/akash-changes.spec.ts b/apps/chain-indexer/src/akash/akash-changes.spec.ts new file mode 100644 index 0000000000..55ca1ef94f --- /dev/null +++ b/apps/chain-indexer/src/akash/akash-changes.spec.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; + +import type { AkashBlockChanges, AkashChangeBody } from "@src/akash/akash-changes"; +import { collectAkashAddresses, isProviderChange } from "@src/akash/akash-changes"; + +const BLOCK_TIME = new Date("2026-08-13T00:00:00Z"); + +describe("collectAkashAddresses", () => { + it("collects owners, providers and depositors from deployment-keyed changes", () => { + const addresses = collectAkashAddresses([ + block([ + { kind: "deploymentCreated", key: { owner: "akash1owner", dseq: "1" }, denom: "uakt", deposit: "1", depositor: "akash1depositor", groups: [] }, + { kind: "bidCreated", key: { owner: "akash1owner", dseq: "1", gseq: 1, oseq: 1, bseq: 0, provider: "akash1bidder" }, price: "1", priceDenom: "uakt" } + ]) + ]); + + expect(addresses).toEqual(new Set(["akash1owner", "akash1depositor", "akash1bidder"])); + }); + + it("collects the owner and auditor from provider and audit changes", () => { + const addresses = collectAkashAddresses([ + block([ + { kind: "providerCreated", owner: "akash1prov", hostUri: "https://x", email: null, website: null, attributes: [] }, + { kind: "providerDeleted", owner: "akash1gone" }, + { kind: "providerAttributesSigned", owner: "akash1prov", auditor: "akash1auditor", attributes: [] }, + { kind: "providerAttributesUnsigned", owner: "akash1prov", auditor: "akash1revoker", keys: [] } + ]) + ]); + + expect(addresses).toEqual(new Set(["akash1prov", "akash1gone", "akash1auditor", "akash1revoker"])); + }); +}); + +describe("isProviderChange", () => { + it("narrows provider and audit kinds and rejects deployment-keyed ones", () => { + const provider: AkashChangeBody = { kind: "providerDeleted", owner: "akash1prov" }; + const deployment: AkashChangeBody = { kind: "deploymentClosed", key: { owner: "akash1owner", dseq: "1" } }; + + expect(isProviderChange({ ...provider, txIndex: 0, msgIndex: 0 })).toBe(true); + expect(isProviderChange({ ...deployment, txIndex: 0, msgIndex: 0 })).toBe(false); + }); +}); + +function block(bodies: AkashChangeBody[]): AkashBlockChanges { + return { height: 100, datetime: BLOCK_TIME, changes: bodies.map((body, index) => ({ ...body, txIndex: 0, msgIndex: index })) }; +} diff --git a/apps/chain-indexer/src/akash/akash-changes.ts b/apps/chain-indexer/src/akash/akash-changes.ts index ad34fe836c..c282358de5 100644 --- a/apps/chain-indexer/src/akash/akash-changes.ts +++ b/apps/chain-indexer/src/akash/akash-changes.ts @@ -41,7 +41,21 @@ interface ChangeOrigin { msgIndex: number | null; } +export interface ProviderAttribute { + key: string; + value: string; +} + +export type ProviderChangeBody = + | { kind: "providerCreated"; owner: string; hostUri: string; email: string | null; website: string | null; attributes: ProviderAttribute[] } + | { kind: "providerUpdated"; owner: string; hostUri: string; email: string | null; website: string | null; attributes: ProviderAttribute[] } + | { kind: "providerDeleted"; owner: string } + | { kind: "providerAttributesSigned"; owner: string; auditor: string; attributes: ProviderAttribute[] } + /** Empty `keys` means the auditor revoked every attribute they signed for this provider. */ + | { kind: "providerAttributesUnsigned"; owner: string; auditor: string; keys: string[] }; + export type AkashChangeBody = + | ProviderChangeBody | { kind: "deploymentCreated"; key: DeploymentKey; denom: string; deposit: string; depositor: string | null; groups: NormalizedGroup[] } | { kind: "deploymentDeposited"; key: DeploymentKey; amount: string; depositor: string | null } | { kind: "deploymentUpdated"; key: DeploymentKey } @@ -61,6 +75,26 @@ export type AkashChange = AkashChangeBody & ChangeOrigin; export type AkashChangeKind = AkashChange["kind"]; +export type ProviderChange = ProviderChangeBody & ChangeOrigin; + +const PROVIDER_REGISTRY_KINDS = new Set(["providerCreated", "providerUpdated", "providerDeleted"]); +const PROVIDER_AUDIT_KINDS = new Set(["providerAttributesSigned", "providerAttributesUnsigned"]); + +export type ProviderRegistryChange = Extract; +export type ProviderAuditChange = Extract; + +export function isProviderRegistryChange(change: AkashChange): change is ProviderRegistryChange { + return PROVIDER_REGISTRY_KINDS.has(change.kind); +} + +export function isProviderAuditChange(change: AkashChange): change is ProviderAuditChange { + return PROVIDER_AUDIT_KINDS.has(change.kind); +} + +export function isProviderChange(change: AkashChange): change is ProviderChange { + return isProviderRegistryChange(change) || isProviderAuditChange(change); +} + /** Everything derived from one block, in the exact order the chain applied it (tx order, then message order, then that tx's close events). */ export interface AkashBlockChanges { height: number; @@ -74,6 +108,13 @@ export function collectAkashAddresses(blocks: AkashBlockChanges[]): Set for (const block of blocks) { for (const change of block.changes) { + if (isProviderChange(change)) { + addresses.add(change.owner); + if ("auditor" in change) { + addresses.add(change.auditor); + } + continue; + } addresses.add(change.key.owner); if ("provider" in change) { addresses.add(change.provider); diff --git a/apps/chain-indexer/src/akash/akash-deriver.spec.ts b/apps/chain-indexer/src/akash/akash-deriver.spec.ts index b29628df0e..eb14ee295c 100644 --- a/apps/chain-indexer/src/akash/akash-deriver.spec.ts +++ b/apps/chain-indexer/src/akash/akash-deriver.spec.ts @@ -30,6 +30,76 @@ describe("deriveAkashChanges", () => { ]); }); + it("derives provider and audit changes alongside deployment ones", () => { + const changes = deriveAkashChanges( + block({ + messages: [ + { + typeUrl: "/akash.provider.v1beta4.MsgCreateProvider", + body: { owner: "akash1owner", hostUri: "https://provider.example.com:8443", attributes: [{ key: "region", value: "us-west" }], info: {} } + }, + { + typeUrl: "/akash.audit.v1.MsgSignProviderAttributes", + body: { owner: "akash1owner", auditor: "akash1auditor", attributes: [{ key: "region", value: "us-west" }] } + } + ] + }) + ); + + expect(changes.changes).toEqual([ + { + kind: "providerCreated", + owner: "akash1owner", + hostUri: "https://provider.example.com:8443", + email: null, + website: null, + attributes: [{ key: "region", value: "us-west" }], + txIndex: 0, + msgIndex: 0 + }, + { + kind: "providerAttributesSigned", + owner: "akash1owner", + auditor: "akash1auditor", + attributes: [{ key: "region", value: "us-west" }], + txIndex: 0, + msgIndex: 1 + } + ]); + }); + + it("unwraps an authz-wrapped audit sign", () => { + const changes = deriveAkashChanges( + block({ + messages: [ + { + typeUrl: "/cosmos.authz.v1beta1.MsgExec", + body: { + grantee: "akash1grantee", + msgs: [ + { + typeUrl: "/akash.audit.v1beta3.MsgSignProviderAttributes", + decoded: { owner: "akash1owner", auditor: "akash1auditor", attributes: [{ key: "tier", value: "community" }] } + } + ] + } + } + ] + }) + ); + + expect(changes.changes).toEqual([ + { + kind: "providerAttributesSigned", + owner: "akash1owner", + auditor: "akash1auditor", + attributes: [{ key: "tier", value: "community" }], + txIndex: 0, + msgIndex: 0 + } + ]); + }); + it("skips messages in failed transactions", () => { const changes = deriveAkashChanges( block({ diff --git a/apps/chain-indexer/src/akash/akash-deriver.ts b/apps/chain-indexer/src/akash/akash-deriver.ts index 6f600ce495..d69315550c 100644 --- a/apps/chain-indexer/src/akash/akash-deriver.ts +++ b/apps/chain-indexer/src/akash/akash-deriver.ts @@ -1,7 +1,9 @@ import type { AkashBlockChanges, AkashChange, AkashChangeBody } from "@src/akash/akash-changes"; import { asInteger, asRecord, asString } from "@src/akash/json"; +import { normalizeAuditMessage } from "@src/akash/normalize-audit"; import { normalizeDeploymentMessage } from "@src/akash/normalize-deployment"; import { normalizeMarketMessage } from "@src/akash/normalize-market"; +import { normalizeProviderMessage } from "@src/akash/normalize-provider"; import { asUint64String } from "@src/akash/uint64"; import type { DecodedBlock, DecodedEvent } from "@src/pipeline/decoded-block"; import { MAX_EXEC_DEPTH, MSG_EXEC_TYPE_URL } from "@src/pipeline/msg-exec"; @@ -11,7 +13,7 @@ const DEPLOYMENT_CLOSED_EVENT_TYPE = "akash.deployment.v1.EventDeploymentClosed" const LEASE_CLOSED_EVENT_TYPE = "akash.market.v1.EventLeaseClosed"; /** - * Extracts the deployment and market lifecycle from a block's messages and close events, in the exact + * Extracts the deployment, market, provider and audit lifecycle from a block's messages and close events, in the exact * order the chain applied it: per transaction, messages first (authz MsgExec unwrapped through the * decoder-provided `decoded` field), then that transaction's close events, which catch deployment and * lease closes happening as side effects (group close, authz revoke, overdraw on withdraw). Messages @@ -56,7 +58,11 @@ function addMessage(changes: AkashChange[], typeUrl: string, body: unknown, txIn return; } - const normalized = normalizeDeploymentMessage(typeUrl, record) ?? normalizeMarketMessage(typeUrl, record); + const normalized = + normalizeDeploymentMessage(typeUrl, record) ?? + normalizeMarketMessage(typeUrl, record) ?? + normalizeProviderMessage(typeUrl, record) ?? + normalizeAuditMessage(typeUrl, record); if (normalized) { changes.push({ ...normalized, txIndex, msgIndex }); diff --git a/apps/chain-indexer/src/akash/akash-writer.service.spec.ts b/apps/chain-indexer/src/akash/akash-writer.service.spec.ts index 2f332f3321..fa1e0fb5d9 100644 --- a/apps/chain-indexer/src/akash/akash-writer.service.spec.ts +++ b/apps/chain-indexer/src/akash/akash-writer.service.spec.ts @@ -95,6 +95,20 @@ describe(AkashWriter.name, () => { expect(inserts).toEqual([]); }); + it("ignores provider changes entirely", async () => { + const { writer, tx, inserts, selects, logger } = setup(); + + await writer.write( + tx, + [block(100, [{ kind: "providerCreated", owner: OWNER, hostUri: "https://x", email: null, website: null, attributes: [] }])], + ACCOUNT_IDS + ); + + expect(inserts).toEqual([]); + expect(selects).toEqual([]); + expect(logger.warn).not.toHaveBeenCalled(); + }); + it("logs orphan references without aborting the batch", async () => { const { writer, tx, logger, inserts } = setup(); diff --git a/apps/chain-indexer/src/akash/akash-writer.service.ts b/apps/chain-indexer/src/akash/akash-writer.service.ts index 1471145dd1..605f4e699b 100644 --- a/apps/chain-indexer/src/akash/akash-writer.service.ts +++ b/apps/chain-indexer/src/akash/akash-writer.service.ts @@ -3,6 +3,7 @@ import groupBy from "lodash/groupBy"; import { inject, singleton } from "tsyringe"; import type { AkashBlockChanges, DeploymentKey, NormalizedResource } from "@src/akash/akash-changes"; +import { isProviderChange } from "@src/akash/akash-changes"; import { decFromString, decToString } from "@src/akash/dec"; import type { BidStateValue, DeploymentAggState, GroupStateValue, ReducerWarning } from "@src/akash/deployment-reducer"; import { applyBlockChanges, stateKey } from "@src/akash/deployment-reducer"; @@ -42,6 +43,9 @@ export class AkashWriter { } const keyed = this.#collectKeys(withChanges, accountIds); + if (keyed.length === 0) { + return; + } const { states, deploymentIds, groupIds, loadedAddressIds } = await this.#loadStates(tx, keyed); const warnings = withChanges.flatMap(block => applyBlockChanges(states, block)); @@ -68,6 +72,9 @@ export class AkashWriter { const byKey = new Map(); for (const block of blocks) { for (const change of block.changes) { + if (isProviderChange(change)) { + continue; + } byKey.set(stateKey(change.key), change.key); } } diff --git a/apps/chain-indexer/src/akash/deployment-reducer.spec.ts b/apps/chain-indexer/src/akash/deployment-reducer.spec.ts index 6ebc1dd3dd..c24ffbec13 100644 --- a/apps/chain-indexer/src/akash/deployment-reducer.spec.ts +++ b/apps/chain-indexer/src/akash/deployment-reducer.spec.ts @@ -166,6 +166,15 @@ describe("applyBlockChanges", () => { expect(warnings).toEqual([{ code: "AKASH_ORPHAN_REFERENCE", kind: "deploymentDeposited", owner: OWNER, dseq: "42", height: 100 }]); }); + it("ignores provider changes without warning or mutating state", () => { + const { states } = setup(); + + const warnings = applyBlockChanges(states, block(100, [{ kind: "providerDeleted", owner: OWNER }])); + + expect(states.size).toBe(0); + expect(warnings).toEqual([]); + }); + it("applies close-event fallbacks with settlement, only when not already closed", () => { const { states } = setup(); diff --git a/apps/chain-indexer/src/akash/deployment-reducer.ts b/apps/chain-indexer/src/akash/deployment-reducer.ts index 59809b9385..6cb1b4dea9 100644 --- a/apps/chain-indexer/src/akash/deployment-reducer.ts +++ b/apps/chain-indexer/src/akash/deployment-reducer.ts @@ -1,4 +1,5 @@ import type { AkashBlockChanges, AkashChange, DeploymentKey, LeaseSlot, NormalizedGroup, NormalizedResource } from "@src/akash/akash-changes"; +import { isProviderChange } from "@src/akash/akash-changes"; import { decCeilInt, decFromInt, decFromString, decQuo, decToString, decTruncateInt } from "@src/akash/dec"; import { normalizeDenom } from "@src/akash/denom"; import { settle, sumLeaseRate } from "@src/akash/settlement"; @@ -109,6 +110,9 @@ export function applyBlockChanges(states: Map, block const decided = new Map(); for (const change of block.changes) { + if (isProviderChange(change)) { + continue; + } const key = stateKey(change.key); if (change.kind === "deploymentCreated") { diff --git a/apps/chain-indexer/src/akash/normalize-audit.spec.ts b/apps/chain-indexer/src/akash/normalize-audit.spec.ts new file mode 100644 index 0000000000..1d5c4563fd --- /dev/null +++ b/apps/chain-indexer/src/akash/normalize-audit.spec.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vitest"; + +import { normalizeAuditMessage } from "@src/akash/normalize-audit"; + +describe("normalizeAuditMessage", () => { + it("normalizes a legacy v1beta1 sign with its attributes", () => { + const change = normalizeAuditMessage("/akash.audit.v1beta1.MsgSignProviderAttributes", { + owner: "akash1owner", + auditor: "akash1auditor", + attributes: [ + { key: "region", value: "us-west" }, + { key: "tier", value: "community" } + ] + }); + + expect(change).toEqual({ + kind: "providerAttributesSigned", + owner: "akash1owner", + auditor: "akash1auditor", + attributes: [ + { key: "region", value: "us-west" }, + { key: "tier", value: "community" } + ] + }); + }); + + it("normalizes a current v1 sign identically to the legacy eras", () => { + const change = normalizeAuditMessage("/akash.audit.v1.MsgSignProviderAttributes", { + owner: "akash1owner", + auditor: "akash1auditor", + attributes: [] + }); + + expect(change).toEqual({ kind: "providerAttributesSigned", owner: "akash1owner", auditor: "akash1auditor", attributes: [] }); + }); + + it("normalizes a keyed delete, dropping non-string keys", () => { + const change = normalizeAuditMessage("/akash.audit.v1beta3.MsgDeleteProviderAttributes", { + owner: "akash1owner", + auditor: "akash1auditor", + keys: ["region", 7, "tier"] + }); + + expect(change).toEqual({ kind: "providerAttributesUnsigned", owner: "akash1owner", auditor: "akash1auditor", keys: ["region", "tier"] }); + }); + + it("normalizes a delete without keys to an empty list meaning delete-all", () => { + const change = normalizeAuditMessage("/akash.audit.v1.MsgDeleteProviderAttributes", { owner: "akash1owner", auditor: "akash1auditor" }); + + expect(change).toEqual({ kind: "providerAttributesUnsigned", owner: "akash1owner", auditor: "akash1auditor", keys: [] }); + }); + + it("returns null when the owner or auditor is missing", () => { + expect(normalizeAuditMessage("/akash.audit.v1.MsgSignProviderAttributes", { owner: "akash1owner" })).toBeNull(); + expect(normalizeAuditMessage("/akash.audit.v1.MsgDeleteProviderAttributes", { auditor: "akash1auditor" })).toBeNull(); + }); + + it("returns null for unrelated type urls", () => { + expect(normalizeAuditMessage("/akash.audit.v1beta4.MsgSignProviderAttributes", { owner: "a", auditor: "b" })).toBeNull(); + expect(normalizeAuditMessage("/akash.provider.v1beta4.MsgCreateProvider", {})).toBeNull(); + }); +}); diff --git a/apps/chain-indexer/src/akash/normalize-audit.ts b/apps/chain-indexer/src/akash/normalize-audit.ts new file mode 100644 index 0000000000..59db22a7c3 --- /dev/null +++ b/apps/chain-indexer/src/akash/normalize-audit.ts @@ -0,0 +1,35 @@ +import type { AkashChangeBody } from "@src/akash/akash-changes"; +import { akashTypeUrlSet } from "@src/akash/akash-changes"; +import { asString } from "@src/akash/json"; +import { attributeList } from "@src/akash/resources"; + +const AUDIT_VERSIONS = ["v1beta1", "v1beta2", "v1beta3", "v1"] as const; + +const SIGN_ATTRIBUTES = typeUrlSet("MsgSignProviderAttributes"); +const DELETE_ATTRIBUTES = typeUrlSet("MsgDeleteProviderAttributes"); + +function typeUrlSet(name: string): Set { + return akashTypeUrlSet("audit", name, AUDIT_VERSIONS); +} + +export function normalizeAuditMessage(typeUrl: string, body: Record): AkashChangeBody | null { + if (SIGN_ATTRIBUTES.has(typeUrl)) { + const identity = auditIdentity(body); + return identity ? { kind: "providerAttributesSigned", ...identity, attributes: attributeList(body.attributes) } : null; + } + if (DELETE_ATTRIBUTES.has(typeUrl)) { + const identity = auditIdentity(body); + return identity ? { kind: "providerAttributesUnsigned", ...identity, keys: stringKeys(body.keys) } : null; + } + return null; +} + +function auditIdentity(body: Record): { owner: string; auditor: string } | null { + const owner = asString(body.owner); + const auditor = asString(body.auditor); + return owner && auditor ? { owner, auditor } : null; +} + +function stringKeys(keys: unknown): string[] { + return Array.isArray(keys) ? keys.filter((key): key is string => typeof key === "string") : []; +} diff --git a/apps/chain-indexer/src/akash/normalize-provider.spec.ts b/apps/chain-indexer/src/akash/normalize-provider.spec.ts new file mode 100644 index 0000000000..954d81df09 --- /dev/null +++ b/apps/chain-indexer/src/akash/normalize-provider.spec.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; + +import { normalizeProviderMessage } from "@src/akash/normalize-provider"; + +describe("normalizeProviderMessage", () => { + it("normalizes a legacy v1beta1 create with info and attributes", () => { + const change = normalizeProviderMessage("/akash.provider.v1beta1.MsgCreateProvider", { + owner: "akash1owner", + hostUri: "https://provider.example.com:8443", + attributes: [{ key: "region", value: "us-west" }], + info: { email: "ops@example.com", website: "https://example.com" } + }); + + expect(change).toEqual({ + kind: "providerCreated", + owner: "akash1owner", + hostUri: "https://provider.example.com:8443", + email: "ops@example.com", + website: "https://example.com", + attributes: [{ key: "region", value: "us-west" }] + }); + }); + + it("normalizes a current v1beta4 update identically to the legacy eras", () => { + const change = normalizeProviderMessage("/akash.provider.v1beta4.MsgUpdateProvider", { + owner: "akash1owner", + hostUri: "https://new.example.com:8443", + attributes: [], + info: { email: "", website: "" } + }); + + expect(change).toEqual({ + kind: "providerUpdated", + owner: "akash1owner", + hostUri: "https://new.example.com:8443", + email: null, + website: null, + attributes: [] + }); + }); + + it("normalizes a create without info or attributes", () => { + const change = normalizeProviderMessage("/akash.provider.v1beta3.MsgCreateProvider", { + owner: "akash1owner", + hostUri: "https://provider.example.com:8443" + }); + + expect(change).toEqual({ + kind: "providerCreated", + owner: "akash1owner", + hostUri: "https://provider.example.com:8443", + email: null, + website: null, + attributes: [] + }); + }); + + it("normalizes a delete", () => { + const change = normalizeProviderMessage("/akash.provider.v1beta2.MsgDeleteProvider", { owner: "akash1owner" }); + + expect(change).toEqual({ kind: "providerDeleted", owner: "akash1owner" }); + }); + + it("returns null when the owner or host uri is missing", () => { + expect(normalizeProviderMessage("/akash.provider.v1beta4.MsgCreateProvider", { hostUri: "https://x" })).toBeNull(); + expect(normalizeProviderMessage("/akash.provider.v1beta4.MsgUpdateProvider", { owner: "akash1owner" })).toBeNull(); + expect(normalizeProviderMessage("/akash.provider.v1beta4.MsgDeleteProvider", {})).toBeNull(); + }); + + it("returns null for unrelated type urls", () => { + expect(normalizeProviderMessage("/akash.provider.v1beta5.MsgCreateProvider", { owner: "akash1owner", hostUri: "https://x" })).toBeNull(); + expect(normalizeProviderMessage("/akash.deployment.v1beta4.MsgCreateDeployment", {})).toBeNull(); + }); +}); diff --git a/apps/chain-indexer/src/akash/normalize-provider.ts b/apps/chain-indexer/src/akash/normalize-provider.ts new file mode 100644 index 0000000000..5a1c157c8b --- /dev/null +++ b/apps/chain-indexer/src/akash/normalize-provider.ts @@ -0,0 +1,46 @@ +import type { AkashChangeBody } from "@src/akash/akash-changes"; +import { akashTypeUrlSet } from "@src/akash/akash-changes"; +import { asRecord, asString } from "@src/akash/json"; +import { attributeList } from "@src/akash/resources"; + +const PROVIDER_VERSIONS = ["v1beta1", "v1beta2", "v1beta3", "v1beta4"] as const; + +const CREATE_PROVIDER = typeUrlSet("MsgCreateProvider"); +const UPDATE_PROVIDER = typeUrlSet("MsgUpdateProvider"); +const DELETE_PROVIDER = typeUrlSet("MsgDeleteProvider"); + +function typeUrlSet(name: string): Set { + return akashTypeUrlSet("provider", name, PROVIDER_VERSIONS); +} + +/** All four provider proto eras share the same message shape, so parsing is version-independent. */ +export function normalizeProviderMessage(typeUrl: string, body: Record): AkashChangeBody | null { + if (CREATE_PROVIDER.has(typeUrl)) { + return normalizeProviderInfo(body, "providerCreated"); + } + if (UPDATE_PROVIDER.has(typeUrl)) { + return normalizeProviderInfo(body, "providerUpdated"); + } + if (DELETE_PROVIDER.has(typeUrl)) { + const owner = asString(body.owner); + return owner ? { kind: "providerDeleted", owner } : null; + } + return null; +} + +function normalizeProviderInfo(body: Record, kind: "providerCreated" | "providerUpdated"): AkashChangeBody | null { + const owner = asString(body.owner); + const hostUri = asString(body.hostUri); + if (!owner || !hostUri) { + return null; + } + const info = asRecord(body.info); + return { + kind, + owner, + hostUri, + email: asString(info?.email) || null, + website: asString(info?.website) || null, + attributes: attributeList(body.attributes) + }; +} diff --git a/apps/chain-indexer/src/akash/provider-writer.service.spec.ts b/apps/chain-indexer/src/akash/provider-writer.service.spec.ts new file mode 100644 index 0000000000..96d6d70c20 --- /dev/null +++ b/apps/chain-indexer/src/akash/provider-writer.service.spec.ts @@ -0,0 +1,309 @@ +import type { SQL } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { describe, expect, it } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import type { AkashBlockChanges, AkashChangeBody } from "@src/akash/akash-changes"; +import { ProviderWriter } from "@src/akash/provider-writer.service"; +import { ProviderAuditSignatures, Providers } from "@src/db/schema"; +import type { ChainTransaction } from "@src/providers/db.provider"; +import type { LoggerService } from "@src/providers/logging.provider"; + +const OWNER = "akash1prov"; +const AUDITOR = "akash1auditor"; +const BLOCK_TIME = new Date("2026-08-13T00:00:00Z"); +const ACCOUNT_IDS = new Map([ + [OWNER, 7], + [AUDITOR, 8] +]); + +describe(ProviderWriter.name, () => { + it("does nothing for blocks without provider changes", async () => { + const { writer, tx, inserts, deletes, selects } = setup(); + + await writer.write(tx, [block(100, [{ kind: "deploymentClosed", key: { owner: OWNER, dseq: "1" } }])], ACCOUNT_IDS); + + expect(inserts).toEqual([]); + expect(deletes).toEqual([]); + expect(selects).toEqual([]); + }); + + it("folds a create, update and delete across blocks into one guarded upsert with the final state", async () => { + const { writer, tx, inserts, upserts } = setup(); + + await writer.write( + tx, + [ + block(100, [created()]), + block(110, [updated({ hostUri: "https://new.example.com:8443", email: "new@example.com", attributes: [{ key: "tier", value: "pro" }] })]), + block(120, [{ kind: "providerDeleted", owner: OWNER }]) + ], + ACCOUNT_IDS + ); + + expect(rowsFor(inserts, Providers)).toEqual([ + { + ownerAccountId: 7, + hostUri: "https://new.example.com:8443", + email: "new@example.com", + website: null, + attributes: [{ key: "tier", value: "pro" }], + lastProcessedHeight: 120, + createdHeight: 100, + updatedHeight: 110, + deletedHeight: 120 + } + ]); + + const upsert = upserts.find(entry => entry.table === Providers); + expect(sqlText(upsert?.config.setWhere as SQL)).toContain('excluded.last_processed_height >= "akash"."providers"."last_processed_height"'); + }); + + it("resets the row when a deleted provider re-registers", async () => { + const { writer, tx, inserts } = setup({ + providers: [providerRow({ lastProcessedHeight: 120, updatedHeight: 110, deletedHeight: 120 })] + }); + + await writer.write(tx, [block(200, [created({ hostUri: "https://back.example.com:8443" })])], ACCOUNT_IDS); + + expect(rowsFor(inserts, Providers)).toEqual([ + expect.objectContaining({ + hostUri: "https://back.example.com:8443", + lastProcessedHeight: 200, + createdHeight: 200, + updatedHeight: null, + deletedHeight: null + }) + ]); + }); + + it("skips blocks at or below the stored watermark", async () => { + const { writer, tx, inserts } = setup({ providers: [providerRow({ lastProcessedHeight: 500 })] }); + + await writer.write(tx, [block(400, [created()]), block(500, [{ kind: "providerDeleted", owner: OWNER }])], ACCOUNT_IDS); + + expect(inserts).toEqual([]); + }); + + it("warns on an update for an unknown provider without writing", async () => { + const { writer, tx, logger, inserts } = setup(); + + await writer.write(tx, [block(100, [updated({})])], ACCOUNT_IDS); + + expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ event: "PROVIDER_ORPHAN_REFERENCE", count: 1 })); + expect(inserts).toEqual([]); + }); + + it("upserts signed attributes deduped last-wins with a height guard", async () => { + const { writer, tx, inserts, upserts } = setup(); + + await writer.write( + tx, + [ + block(100, [ + { + kind: "providerAttributesSigned", + owner: OWNER, + auditor: AUDITOR, + attributes: [ + { key: "region", value: "us-east" }, + { key: "region", value: "us-west" }, + { key: "tier", value: "community" } + ] + } + ]) + ], + ACCOUNT_IDS + ); + + expect(rowsFor(inserts, ProviderAuditSignatures)).toEqual([ + { ownerAccountId: 7, auditorAccountId: 8, key: "region", value: "us-west", height: 100 }, + { ownerAccountId: 7, auditorAccountId: 8, key: "tier", value: "community", height: 100 } + ]); + + const upsert = upserts.find(entry => entry.table === ProviderAuditSignatures); + expect(sqlText(upsert?.config.setWhere as SQL)).toContain('excluded.height >= "akash"."provider_audit_signatures"."height"'); + }); + + it("deletes the given keys with a height guard, and all of the auditor's keys when none are given", async () => { + const { writer, tx, deletes } = setup(); + + await writer.write( + tx, + [ + block(100, [ + { kind: "providerAttributesUnsigned", owner: OWNER, auditor: AUDITOR, keys: ["region", "tier"] }, + { kind: "providerAttributesUnsigned", owner: OWNER, auditor: AUDITOR, keys: [] } + ]) + ], + ACCOUNT_IDS + ); + + expect(deletes).toHaveLength(2); + const [keyedDelete, deleteAll] = deletes.map(entry => sqlText(entry.where as SQL)); + expect(keyedDelete).toContain('"owner_account_id" = '); + expect(keyedDelete).toContain('"auditor_account_id" = '); + expect(keyedDelete).toContain('"height" <= '); + expect(keyedDelete).toContain('"key" in '); + expect(deleteAll).toContain('"height" <= '); + expect(deleteAll).not.toContain('"key" in '); + }); + + it("applies audit changes even when the provider was never registered", async () => { + const { writer, tx, inserts, logger } = setup(); + + await writer.write( + tx, + [block(100, [{ kind: "providerAttributesSigned", owner: OWNER, auditor: AUDITOR, attributes: [{ key: "region", value: "us-west" }] }])], + ACCOUNT_IDS + ); + + expect(rowsFor(inserts, ProviderAuditSignatures)).toHaveLength(1); + expect(logger.warn).not.toHaveBeenCalled(); + }); + + it("locks provider state rows for update ordered by owner account id", async () => { + const { writer, tx, stateLocks } = setup(); + + await writer.write(tx, [block(100, [created()])], ACCOUNT_IDS); + + expect(stateLocks).toHaveLength(1); + expect(stateLocks[0].orderBy).toHaveLength(1); + expect(stateLocks[0].orderBy[0]).toBe(Providers.ownerAccountId); + expect(stateLocks[0].for).toEqual(["update"]); + }); + + it("serializes audit writes under a single transaction-scoped advisory lock", async () => { + const { writer, tx, executes } = setup(); + + await writer.write( + tx, + [ + block(100, [{ kind: "providerAttributesSigned", owner: OWNER, auditor: AUDITOR, attributes: [{ key: "region", value: "us-west" }] }]), + block(101, [{ kind: "providerAttributesUnsigned", owner: OWNER, auditor: AUDITOR, keys: ["region"] }]) + ], + ACCOUNT_IDS + ); + + expect(executes).toHaveLength(1); + expect(sqlText(executes[0])).toContain("pg_advisory_xact_lock"); + }); + + it("does not take the audit advisory lock when the batch has no audit changes", async () => { + const { writer, tx, executes } = setup(); + + await writer.write(tx, [block(100, [created()])], ACCOUNT_IDS); + + expect(executes).toEqual([]); + }); + + function setup(input?: { providers?: Record[] }) { + const inserts: { table: unknown; rows: Record[] }[] = []; + const upserts: { table: unknown; config: Record }[] = []; + const deletes: { table: unknown; where: unknown }[] = []; + const selects: unknown[] = []; + const executes: SQL[] = []; + const stateLocks: { orderBy: unknown[]; for: unknown[] }[] = []; + + const selectChain = () => { + const lock: { orderBy: unknown[]; for: unknown[] } = { orderBy: [], for: [] }; + stateLocks.push(lock); + const chain = { + where: () => chain, + orderBy: (...args: unknown[]) => { + lock.orderBy = args; + return chain; + }, + for: (...args: unknown[]) => { + lock.for = args; + return chain; + }, + then: (resolve: (rows: unknown[]) => unknown, reject?: (error: unknown) => unknown) => Promise.resolve(input?.providers ?? []).then(resolve, reject) + }; + return chain; + }; + + const tx = { + insert: (table: unknown) => ({ + values: (rows: Record | Record[]) => { + const rowArray = Array.isArray(rows) ? rows : [rows]; + inserts.push({ table, rows: rowArray }); + return Object.assign(Promise.resolve(), { + onConflictDoUpdate: (config: Record) => { + upserts.push({ table, config }); + return Promise.resolve(); + } + }); + } + }), + select: (fields?: unknown) => { + selects.push(fields); + return { from: () => selectChain() }; + }, + delete: (table: unknown) => ({ + where: (condition: unknown) => { + deletes.push({ table, where: condition }); + return Promise.resolve(); + } + }), + execute: (query: SQL) => { + executes.push(query); + return Promise.resolve(); + } + }; + + const logger = mock(); + return { writer: new ProviderWriter(logger), tx: tx as unknown as ChainTransaction, inserts, upserts, deletes, selects, executes, stateLocks, logger }; + } + + function providerRow(overrides: Record) { + return { + ownerAccountId: 7, + hostUri: "https://provider.example.com:8443", + email: null, + website: null, + attributes: [], + lastProcessedHeight: 100, + createdHeight: 100, + updatedHeight: null, + deletedHeight: null, + ...overrides + }; + } + + function created(overrides?: Partial>): AkashChangeBody { + return { + kind: "providerCreated", + owner: OWNER, + hostUri: "https://provider.example.com:8443", + email: null, + website: null, + attributes: [{ key: "region", value: "us-west" }], + ...overrides + }; + } + + function updated(overrides: Partial>): AkashChangeBody { + return { + kind: "providerUpdated", + owner: OWNER, + hostUri: "https://provider.example.com:8443", + email: null, + website: null, + attributes: [], + ...overrides + }; + } + + function block(height: number, bodies: AkashChangeBody[]): AkashBlockChanges { + return { height, datetime: BLOCK_TIME, changes: bodies.map((body, index) => ({ ...body, txIndex: 0, msgIndex: index })) }; + } + + function rowsFor(inserts: { table: unknown; rows: Record[] }[], table: unknown): Record[] { + return inserts.filter(insert => insert.table === table).flatMap(insert => insert.rows); + } + + function sqlText(query: SQL): string { + return new PgDialect().sqlToQuery(query).sql; + } +}); diff --git a/apps/chain-indexer/src/akash/provider-writer.service.ts b/apps/chain-indexer/src/akash/provider-writer.service.ts new file mode 100644 index 0000000000..f38198f0e3 --- /dev/null +++ b/apps/chain-indexer/src/akash/provider-writer.service.ts @@ -0,0 +1,280 @@ +import { and, eq, inArray, lte, sql } from "drizzle-orm"; +import chunk from "lodash/chunk"; +import { inject, singleton } from "tsyringe"; + +import type { AkashBlockChanges, ProviderAttribute, ProviderChange } from "@src/akash/akash-changes"; +import { isProviderAuditChange, isProviderChange, isProviderRegistryChange } from "@src/akash/akash-changes"; +import { INSERT_CHUNK_SIZE } from "@src/db/insert-chunk-size"; +import { ProviderAuditSignatures, Providers } from "@src/db/schema"; +import { sqlExcluded } from "@src/db/sql-excluded"; +import type { ChainTransaction } from "@src/providers/db.provider"; +import { LoggerService } from "@src/providers/logging.provider"; + +/** + * Serializes the audit-signature section across concurrent committers (overlapping sync pods during a + * rolling deploy, or sync racing a backfill). Writers apply signs and deletes in block order, so two + * writers with different block windows can lock the same (owner, auditor, key) rows in opposite orders + * and deadlock; the provider path avoids this with an ordered `FOR UPDATE`, but audit's delete-all-keys + * touches rows that cannot be pre-locked by key. One transaction-scoped advisory lock removes the + * hazard, and sparse audit traffic makes it near-free in steady state. Shares Postgres's advisory-lock + * namespace with MIGRATION_LOCK_KEY (db.provider.ts), so the value must stay distinct from it. + */ +const AUDIT_SIGNATURE_LOCK_KEY = 7_431_001; + +interface ProviderBlockChanges { + height: number; + changes: ProviderChange[]; +} + +interface ProviderState { + ownerAccountId: number; + hostUri: string; + email: string | null; + website: string | null; + attributes: ProviderAttribute[]; + lastProcessedHeight: number; + createdHeight: number; + updatedHeight: number | null; + deletedHeight: number | null; + touched: boolean; +} + +interface ProviderWarning { + code: "PROVIDER_ORPHAN_REFERENCE"; + kind: ProviderChange["kind"]; + owner: string; + height: number; +} + +/** + * Persists the provider registry and audited attributes inside the block transaction, mirroring the + * fold-then-flush shape of AkashWriter for the aggregate keyed by owner instead of (owner, dseq). + * Provider rows are locked FOR UPDATE in ascending owner-account order and flushed with the + * `last_processed_height` watermark guard, so overlapping writers replaying the same blocks stay + * idempotent. Audit signatures skip the fold: each sign/unsign applies in block order with a per-row + * height guard, serialized by a transaction-scoped advisory lock so overlapping writers can't lock + * audit rows in conflicting orders. x/audit state is independent of x/provider and audit traffic is + * sparse, so folding it is unnecessary and the lock is near-free. + */ +@singleton() +export class ProviderWriter { + readonly #logger: LoggerService; + + constructor(@inject(LoggerService) logger: LoggerService) { + this.#logger = logger; + this.#logger.setContext("PROVIDER_WRITER"); + } + + async write(tx: ChainTransaction, blocks: AkashBlockChanges[], accountIds: Map): Promise { + const withChanges = blocks + .map(block => ({ height: block.height, changes: block.changes.filter(isProviderChange) })) + .filter(block => block.changes.length > 0); + if (withChanges.length === 0) { + return; + } + + await this.#writeProviders(tx, withChanges, accountIds); + await this.#writeAuditSignatures(tx, withChanges, accountIds); + } + + async #writeProviders(tx: ChainTransaction, blocks: ProviderBlockChanges[], accountIds: Map): Promise { + const ownerIds = this.#collectOwnerIds(blocks, accountIds); + if (ownerIds.size === 0) { + return; + } + + const states = await this.#loadStates( + tx, + [...ownerIds.values()].sort((a, b) => a - b) + ); + const warnings: ProviderWarning[] = []; + + for (const block of blocks) { + this.#applyBlockChanges(states, block, ownerIds, warnings); + } + this.#logWarnings(warnings); + + const touched = [...states.values()].filter(state => state.touched); + if (touched.length === 0) { + return; + } + + await tx + .insert(Providers) + .values(touched.map(({ touched: _, ...row }) => row)) + .onConflictDoUpdate({ + target: Providers.ownerAccountId, + set: { + hostUri: sqlExcluded("host_uri"), + email: sqlExcluded("email"), + website: sqlExcluded("website"), + attributes: sqlExcluded("attributes"), + lastProcessedHeight: sqlExcluded("last_processed_height"), + createdHeight: sqlExcluded("created_height"), + updatedHeight: sqlExcluded("updated_height"), + deletedHeight: sqlExcluded("deleted_height") + }, + setWhere: sql`excluded.last_processed_height >= ${Providers.lastProcessedHeight}` + }); + } + + #applyBlockChanges(states: Map, block: ProviderBlockChanges, ownerIds: Map, warnings: ProviderWarning[]): void { + const skippedOwners = this.#ownersAtOrPastWatermark(states, block.height); + + for (const change of block.changes) { + if (isProviderAuditChange(change)) { + continue; + } + const ownerAccountId = this.#requireId(ownerIds, change.owner); + if (skippedOwners.has(ownerAccountId)) { + continue; + } + + const state = states.get(ownerAccountId); + if (change.kind === "providerCreated") { + states.set(ownerAccountId, { + ownerAccountId, + hostUri: change.hostUri, + email: change.email, + website: change.website, + attributes: change.attributes, + lastProcessedHeight: block.height, + createdHeight: block.height, + updatedHeight: null, + deletedHeight: null, + touched: true + }); + continue; + } + + if (!state) { + warnings.push({ code: "PROVIDER_ORPHAN_REFERENCE", kind: change.kind, owner: change.owner, height: block.height }); + continue; + } + + if (change.kind === "providerUpdated") { + state.hostUri = change.hostUri; + state.email = change.email; + state.website = change.website; + state.attributes = change.attributes; + state.updatedHeight = block.height; + } else { + state.deletedHeight = block.height; + } + state.lastProcessedHeight = block.height; + state.touched = true; + } + } + + /** Providers already at or past this block's height saw it in a previous commit; the whole block is a duplicate for them. */ + #ownersAtOrPastWatermark(states: Map, height: number): Set { + const skipped = new Set(); + for (const state of states.values()) { + if (state.lastProcessedHeight >= height) { + skipped.add(state.ownerAccountId); + } + } + return skipped; + } + + async #writeAuditSignatures(tx: ChainTransaction, blocks: ProviderBlockChanges[], accountIds: Map): Promise { + if (!blocks.some(block => block.changes.some(isProviderAuditChange))) { + return; + } + await tx.execute(sql`SELECT pg_advisory_xact_lock(${AUDIT_SIGNATURE_LOCK_KEY})`); + + for (const block of blocks) { + for (const change of block.changes) { + if (change.kind === "providerAttributesSigned") { + await this.#upsertSignatures(tx, change, block.height, accountIds); + } else if (change.kind === "providerAttributesUnsigned") { + await this.#deleteSignatures(tx, change, block.height, accountIds); + } + } + } + } + + async #upsertSignatures( + tx: ChainTransaction, + change: Extract, + height: number, + accountIds: Map + ): Promise { + const ownerAccountId = this.#requireId(accountIds, change.owner); + const auditorAccountId = this.#requireId(accountIds, change.auditor); + const rows = dedupeByKeyLastWins(change.attributes).map(attribute => ({ + ownerAccountId, + auditorAccountId, + key: attribute.key, + value: attribute.value, + height + })); + + for (const rowChunk of chunk(rows, INSERT_CHUNK_SIZE)) { + await tx + .insert(ProviderAuditSignatures) + .values(rowChunk) + .onConflictDoUpdate({ + target: [ProviderAuditSignatures.ownerAccountId, ProviderAuditSignatures.auditorAccountId, ProviderAuditSignatures.key], + set: { value: sqlExcluded("value"), height: sqlExcluded("height") }, + setWhere: sql`excluded.height >= ${ProviderAuditSignatures.height}` + }); + } + } + + /** The `height <=` guard keeps a replayed delete from removing a signature re-signed at a later height. */ + async #deleteSignatures( + tx: ChainTransaction, + change: Extract, + height: number, + accountIds: Map + ): Promise { + const identity = and( + eq(ProviderAuditSignatures.ownerAccountId, this.#requireId(accountIds, change.owner)), + eq(ProviderAuditSignatures.auditorAccountId, this.#requireId(accountIds, change.auditor)), + lte(ProviderAuditSignatures.height, height) + ); + await tx.delete(ProviderAuditSignatures).where(change.keys.length > 0 ? and(identity, inArray(ProviderAuditSignatures.key, change.keys)) : identity); + } + + #collectOwnerIds(blocks: ProviderBlockChanges[], accountIds: Map): Map { + const ownerIds = new Map(); + for (const block of blocks) { + for (const change of block.changes) { + if (isProviderRegistryChange(change)) { + ownerIds.set(change.owner, this.#requireId(accountIds, change.owner)); + } + } + } + return ownerIds; + } + + async #loadStates(tx: ChainTransaction, ownerAccountIds: number[]): Promise> { + const rows = await tx.select().from(Providers).where(inArray(Providers.ownerAccountId, ownerAccountIds)).orderBy(Providers.ownerAccountId).for("update"); + + return new Map(rows.map(row => [row.ownerAccountId, { ...row, touched: false }])); + } + + #logWarnings(warnings: ProviderWarning[]): void { + if (warnings.length === 0) { + return; + } + this.#logger.warn({ event: "PROVIDER_ORPHAN_REFERENCE", count: warnings.length, samples: warnings.slice(0, 5) }); + } + + #requireId(accountIds: Map, address: string): number { + const id = accountIds.get(address); + if (id === undefined) { + throw new Error(`No interned account id for address ${address}`); + } + return id; + } +} + +/** + * A message may sign the same key twice; feeding both rows to one ON CONFLICT DO UPDATE would raise + * `21000: command cannot affect row a second time`, so collapse to the last occurrence. + */ +function dedupeByKeyLastWins(attributes: ProviderAttribute[]): ProviderAttribute[] { + return [...new Map(attributes.map(attribute => [attribute.key, attribute])).values()]; +} diff --git a/apps/chain-indexer/src/akash/resources.ts b/apps/chain-indexer/src/akash/resources.ts index b9f813c882..f56954a3cf 100644 --- a/apps/chain-indexer/src/akash/resources.ts +++ b/apps/chain-indexer/src/akash/resources.ts @@ -1,4 +1,4 @@ -import type { NormalizedGroup, NormalizedResource } from "@src/akash/akash-changes"; +import type { NormalizedGroup, NormalizedResource, ProviderAttribute } from "@src/akash/akash-changes"; import { asRecord } from "@src/akash/json"; /** @@ -75,7 +75,7 @@ function gpuAttributes(gpu: Record | null): { vendor: string | return { vendor: match[1], model: match[2] !== "*" ? match[2] : null }; } -function attributeList(attributes: unknown): Array<{ key: string; value: string }> { +export function attributeList(attributes: unknown): ProviderAttribute[] { if (!Array.isArray(attributes)) { return []; } diff --git a/apps/chain-indexer/src/db/schema.spec.ts b/apps/chain-indexer/src/db/schema.spec.ts index ef9abcd1a9..3ddc63b16e 100644 --- a/apps/chain-indexer/src/db/schema.spec.ts +++ b/apps/chain-indexer/src/db/schema.spec.ts @@ -17,6 +17,8 @@ import { ProposalDeposits, Proposals, ProposalVotes, + ProviderAuditSignatures, + Providers, UnbondingDelegations, Validators } from "@src/db/schema"; @@ -184,3 +186,25 @@ describe("akash deployment schema", () => { expect(config.columns.find(column => column.name === "tx_index")?.notNull).toBe(false); }); }); + +describe("akash provider schema", () => { + it("keys providers by their owner account with lifecycle heights and the replay watermark", () => { + const config = getTableConfig(Providers); + + expect(config.name).toBe("providers"); + expect(config.columns.find(column => column.name === "owner_account_id")?.primary).toBe(true); + expect(config.foreignKeys[0].reference().foreignColumns[0].name).toBe("id"); + expect(config.columns.map(column => column.name)).toEqual( + expect.arrayContaining(["host_uri", "email", "website", "attributes", "last_processed_height", "created_height", "updated_height", "deleted_height"]) + ); + }); + + it("keys audit signatures by owner, auditor and key with account foreign keys for both parties", () => { + const config = getTableConfig(ProviderAuditSignatures); + + expect(config.primaryKeys[0].columns.map(column => column.name)).toEqual(["owner_account_id", "auditor_account_id", "key"]); + expect(config.foreignKeys).toHaveLength(2); + config.foreignKeys.forEach(foreignKey => expect(foreignKey.reference().foreignColumns[0].name).toBe("id")); + expect(config.columns.map(column => column.name)).toEqual(expect.arrayContaining(["value", "height"])); + }); +}); diff --git a/apps/chain-indexer/src/db/schema.ts b/apps/chain-indexer/src/db/schema.ts index 2290be387f..99fcb2dd0b 100644 --- a/apps/chain-indexer/src/db/schema.ts +++ b/apps/chain-indexer/src/db/schema.ts @@ -16,6 +16,7 @@ import { uniqueIndex } from "drizzle-orm/pg-core"; +import type { ProviderAttribute } from "@src/akash/akash-changes"; import { bytea } from "@src/db/bytea"; export const cosmosSchema = pgSchema("cosmos"); @@ -505,3 +506,48 @@ export const DeploymentEvents = akashSchema.table( }, t => [primaryKey({ columns: [t.deploymentId, t.height, t.ordinal] })] ); + +/** + * Current on-chain provider state, one row per owner — mirroring `akash query provider list`. + * A provider that deletes and re-registers reuses its row: create resets `created_height` and + * clears `updated_height`/`deleted_height`, so the original registration height is not kept. + * Attributes are the full replace-on-update set from MsgCreate/MsgUpdateProvider. + * `last_processed_height` is the replay watermark (same semantics as deployments). + */ +export const Providers = akashSchema.table("providers", { + ownerAccountId: integer("owner_account_id") + .primaryKey() + .references(() => Accounts.id), + hostUri: text("host_uri").notNull(), + email: text("email"), + website: text("website"), + attributes: jsonb("attributes").$type().notNull(), + lastProcessedHeight: bigint("last_processed_height", { mode: "number" }).notNull(), + createdHeight: bigint("created_height", { mode: "number" }).notNull(), + updatedHeight: bigint("updated_height", { mode: "number" }), + deletedHeight: bigint("deleted_height", { mode: "number" }) +}); + +/** + * Audited provider attributes, one row per (owner, auditor, key) — mirroring the x/audit store. + * Keyed by account rather than the providers row because x/audit never consults x/provider: + * signatures survive provider deletion and can precede registration. MsgSignProviderAttributes + * merges per-key; MsgDeleteProviderAttributes deletes the given keys, or all of the auditor's + * keys when none are given. `height` is the per-row replay guard: signs only apply at or above + * it and deletes only remove rows written at or below the deleting block. + */ +export const ProviderAuditSignatures = akashSchema.table( + "provider_audit_signatures", + { + ownerAccountId: integer("owner_account_id") + .notNull() + .references(() => Accounts.id), + auditorAccountId: integer("auditor_account_id") + .notNull() + .references(() => Accounts.id), + key: text("key").notNull(), + value: text("value").notNull(), + height: bigint("height", { mode: "number" }).notNull() + }, + t => [primaryKey({ columns: [t.ownerAccountId, t.auditorAccountId, t.key] })] +); diff --git a/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts b/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts index b1e32e3c35..eb276ac896 100644 --- a/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts +++ b/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts @@ -4,6 +4,7 @@ import { describe, expect, it } from "vitest"; import { mock } from "vitest-mock-extended"; import type { AkashWriter } from "@src/akash/akash-writer.service"; +import type { ProviderWriter } from "@src/akash/provider-writer.service"; import { AccountTxs, Blocks, IndexerState, MessageDeadLetters, Messages, MessageTypes } from "@src/db/schema"; import type { GovWriter } from "@src/gov/gov-writer.service"; import type { AccountInterner } from "@src/pipeline/balance/account-interner.service"; @@ -255,6 +256,18 @@ describe(BlockCommitterService.name, () => { const accountIds = akashWriter.write.mock.calls[0][2]; expect(accountIds.get("akash1owner")).toBeDefined(); }); + + it("hands the provider writer the same derived changes and account ids as the akash writer", async () => { + const { committer, akashWriter, providerWriter } = setup({ selectResults: [[{ id: 7, type: MSG_SEND }]] }); + + await committer.commit( + buildBlock([MSG_SEND], 10, { + events: [{ type: "akash.deployment.v1.EventDeploymentClosed", attributes: { id: '{"owner":"akash1owner","dseq":"42"}' } }] + }) + ); + + expect(providerWriter.write).toHaveBeenCalledWith(expect.anything(), akashWriter.write.mock.calls[0][1], akashWriter.write.mock.calls[0][2]); + }); }); function setup(input?: { @@ -307,9 +320,10 @@ describe(BlockCommitterService.name, () => { const govWriter = mock(); const akashWriter = mock(); + const providerWriter = mock(); const logger = mock(); - const committer = new BlockCommitterService(dbFake as unknown as ChainDatabase, interner, balanceWriter, govWriter, akashWriter, logger); - return { committer, insertedRows, conflictUpdates, deletions, interner, balanceWriter, govWriter, akashWriter, logger }; + const committer = new BlockCommitterService(dbFake as unknown as ChainDatabase, interner, balanceWriter, govWriter, akashWriter, providerWriter, logger); + return { committer, insertedRows, conflictUpdates, deletions, interner, balanceWriter, govWriter, akashWriter, providerWriter, logger }; } function buildBlock( diff --git a/apps/chain-indexer/src/pipeline/block-committer.service.ts b/apps/chain-indexer/src/pipeline/block-committer.service.ts index d9568d28a8..a1aec9c2b9 100644 --- a/apps/chain-indexer/src/pipeline/block-committer.service.ts +++ b/apps/chain-indexer/src/pipeline/block-committer.service.ts @@ -6,6 +6,7 @@ import type { AkashBlockChanges } from "@src/akash/akash-changes"; import { collectAkashAddresses } from "@src/akash/akash-changes"; import { deriveAkashChanges } from "@src/akash/akash-deriver"; import { AkashWriter } from "@src/akash/akash-writer.service"; +import { ProviderWriter } from "@src/akash/provider-writer.service"; import { INSERT_CHUNK_SIZE } from "@src/db/insert-chunk-size"; import { insertChunked } from "@src/db/insert-chunked"; import { AccountTxs, Blocks, IndexerState, MessageDeadLetters, Messages, MessageTypes, Transactions } from "@src/db/schema"; @@ -47,6 +48,7 @@ export class BlockCommitterService { readonly #balanceWriter: BalanceWriter; readonly #govWriter: GovWriter; readonly #akashWriter: AkashWriter; + readonly #providerWriter: ProviderWriter; readonly #logger: LoggerService; readonly #moduleRegistry = buildModuleAddressRegistry(); readonly #typeIds = new Map(); @@ -57,6 +59,7 @@ export class BlockCommitterService { @inject(BalanceWriter) balanceWriter: BalanceWriter, @inject(GovWriter) govWriter: GovWriter, @inject(AkashWriter) akashWriter: AkashWriter, + @inject(ProviderWriter) providerWriter: ProviderWriter, @inject(LoggerService) logger: LoggerService ) { this.#db = db; @@ -64,6 +67,7 @@ export class BlockCommitterService { this.#balanceWriter = balanceWriter; this.#govWriter = govWriter; this.#akashWriter = akashWriter; + this.#providerWriter = providerWriter; this.#logger = logger; this.#logger.setContext("COMMITTER"); } @@ -148,6 +152,7 @@ export class BlockCommitterService { await insertChunked(tx, AccountTxs, accountTxRows); await this.#govWriter.writeForBlocks(tx, blocks, accountIds); await this.#akashWriter.write(tx, akashChanges, accountIds); + await this.#providerWriter.write(tx, akashChanges, accountIds); await tx .insert(IndexerState) From 17d2fc877c7ffbfbf992fcb49a8cf38d651fb5c2 Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Mon, 17 Aug 2026 23:36:50 +0530 Subject: [PATCH 11/14] feat(indexer): akash bme event handlers with exact ledger records (#3603) * feat(indexer): akash bme event handlers with exact ledger records * refactor(indexer): dedupe json-record and account-id helpers in bme handlers --- .../drizzle/0008_good_galactus.sql | 58 + .../drizzle/meta/0008_snapshot.json | 2759 +++++++++++++++++ apps/chain-indexer/drizzle/meta/_journal.json | 7 + apps/chain-indexer/src/akash/akash-deriver.ts | 18 +- .../src/akash/akash-writer.service.ts | 23 +- apps/chain-indexer/src/akash/json.ts | 12 + .../src/akash/provider-writer.service.ts | 21 +- .../chain-indexer/src/bme/bme-deriver.spec.ts | 371 +++ apps/chain-indexer/src/bme/bme-deriver.ts | 282 ++ .../src/bme/bme-writer.service.spec.ts | 241 ++ .../src/bme/bme-writer.service.ts | 111 + apps/chain-indexer/src/db/schema.ts | 94 + .../balance/account-interner.service.ts | 9 + .../pipeline/block-committer.service.spec.ts | 44 +- .../src/pipeline/block-committer.service.ts | 35 +- .../pipeline/block-decoder.service.spec.ts | 22 + .../src/pipeline/block-decoder.service.ts | 11 +- 17 files changed, 4054 insertions(+), 64 deletions(-) create mode 100644 apps/chain-indexer/drizzle/0008_good_galactus.sql create mode 100644 apps/chain-indexer/drizzle/meta/0008_snapshot.json create mode 100644 apps/chain-indexer/src/bme/bme-deriver.spec.ts create mode 100644 apps/chain-indexer/src/bme/bme-deriver.ts create mode 100644 apps/chain-indexer/src/bme/bme-writer.service.spec.ts create mode 100644 apps/chain-indexer/src/bme/bme-writer.service.ts diff --git a/apps/chain-indexer/drizzle/0008_good_galactus.sql b/apps/chain-indexer/drizzle/0008_good_galactus.sql new file mode 100644 index 0000000000..4680a79833 --- /dev/null +++ b/apps/chain-indexer/drizzle/0008_good_galactus.sql @@ -0,0 +1,58 @@ +CREATE TYPE "akash"."bme_mint_status" AS ENUM('mint_status_unspecified', 'mint_status_healthy', 'mint_status_warning', 'mint_status_halt_cr', 'mint_status_halt_oracle');--> statement-breakpoint +CREATE TABLE "akash"."bme_canceled_records" ( + "denom" text NOT NULL, + "to_denom" text NOT NULL, + "source" text NOT NULL, + "record_height" bigint NOT NULL, + "sequence" bigint NOT NULL, + "height" bigint NOT NULL, + "tx_index" integer, + "cancel_reason" text NOT NULL, + "owner_account_id" integer NOT NULL, + "to_account_id" integer NOT NULL, + "coins_to_burn_denom" text, + "coins_to_burn_amount" numeric(38, 0), + "denom_to_mint" text NOT NULL, + CONSTRAINT "bme_canceled_records_record_height_sequence_denom_to_denom_source_pk" PRIMARY KEY("record_height","sequence","denom","to_denom","source") +); +--> statement-breakpoint +CREATE TABLE "akash"."bme_ledger_records" ( + "denom" text NOT NULL, + "to_denom" text NOT NULL, + "source" text NOT NULL, + "record_height" bigint NOT NULL, + "sequence" bigint NOT NULL, + "height" bigint NOT NULL, + "tx_index" integer, + "burned_from_account_id" integer NOT NULL, + "minted_to_account_id" integer NOT NULL, + "burned_denom" text, + "burned_amount" numeric(38, 0) DEFAULT '0' NOT NULL, + "burned_price" numeric(38, 18), + "minted_denom" text, + "minted_amount" numeric(38, 0) DEFAULT '0' NOT NULL, + "minted_price" numeric(38, 18), + "spread_denom" text, + "spread_amount" numeric(38, 0), + "remint_credit_issued_amount" numeric(38, 0), + "remint_credit_accrued_amount" numeric(38, 0), + CONSTRAINT "bme_ledger_records_record_height_sequence_denom_to_denom_source_pk" PRIMARY KEY("record_height","sequence","denom","to_denom","source") +); +--> statement-breakpoint +CREATE TABLE "akash"."bme_status_changes" ( + "height" bigint NOT NULL, + "ordinal" integer NOT NULL, + "previous_status" "akash"."bme_mint_status" NOT NULL, + "new_status" "akash"."bme_mint_status" NOT NULL, + "collateral_ratio" numeric(38, 18) NOT NULL, + CONSTRAINT "bme_status_changes_height_ordinal_pk" PRIMARY KEY("height","ordinal") +); +--> statement-breakpoint +ALTER TABLE "akash"."bme_canceled_records" ADD CONSTRAINT "bme_canceled_records_owner_account_id_accounts_id_fk" FOREIGN KEY ("owner_account_id") REFERENCES "cosmos"."accounts"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "akash"."bme_canceled_records" ADD CONSTRAINT "bme_canceled_records_to_account_id_accounts_id_fk" FOREIGN KEY ("to_account_id") REFERENCES "cosmos"."accounts"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "akash"."bme_ledger_records" ADD CONSTRAINT "bme_ledger_records_burned_from_account_id_accounts_id_fk" FOREIGN KEY ("burned_from_account_id") REFERENCES "cosmos"."accounts"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "akash"."bme_ledger_records" ADD CONSTRAINT "bme_ledger_records_minted_to_account_id_accounts_id_fk" FOREIGN KEY ("minted_to_account_id") REFERENCES "cosmos"."accounts"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "bme_canceled_records_height_idx" ON "akash"."bme_canceled_records" USING btree ("height");--> statement-breakpoint +CREATE INDEX "bme_ledger_records_height_idx" ON "akash"."bme_ledger_records" USING btree ("height");--> statement-breakpoint +CREATE INDEX "bme_ledger_records_burned_denom_height_idx" ON "akash"."bme_ledger_records" USING btree ("burned_denom","height");--> statement-breakpoint +CREATE INDEX "bme_ledger_records_minted_denom_height_idx" ON "akash"."bme_ledger_records" USING btree ("minted_denom","height"); \ No newline at end of file diff --git a/apps/chain-indexer/drizzle/meta/0008_snapshot.json b/apps/chain-indexer/drizzle/meta/0008_snapshot.json new file mode 100644 index 0000000000..88fa96e1b6 --- /dev/null +++ b/apps/chain-indexer/drizzle/meta/0008_snapshot.json @@ -0,0 +1,2759 @@ +{ + "id": "2c31f1a9-1835-449a-899e-f6eaa1ebec5b", + "prevId": "a2860b14-74a0-48d3-825f-dc6cecbee0d3", + "version": "7", + "dialect": "postgresql", + "tables": { + "cosmos.account_balances": { + "name": "account_balances", + "schema": "cosmos", + "columns": { + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "amount": { + "name": "amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "account_balances_account_id_accounts_id_fk": { + "name": "account_balances_account_id_accounts_id_fk", + "tableFrom": "account_balances", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_balances_account_id_denom_pk": { + "name": "account_balances_account_id_denom_pk", + "columns": [ + "account_id", + "denom" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.account_txs": { + "name": "account_txs", + "schema": "cosmos", + "columns": { + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "account_tx_role", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "account_txs_account_id_accounts_id_fk": { + "name": "account_txs_account_id_accounts_id_fk", + "tableFrom": "account_txs", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_txs_account_id_height_tx_index_role_pk": { + "name": "account_txs_account_id_height_tx_index_role_pk", + "columns": [ + "account_id", + "height", + "tx_index", + "role" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.accounts": { + "name": "accounts", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_number": { + "name": "account_number", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "account_type": { + "name": "account_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_module_account": { + "name": "is_module_account", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": { + "accounts_address_idx": { + "name": "accounts_address_idx", + "columns": [ + { + "expression": "address", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.balance_changes": { + "name": "balance_changes", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delta": { + "name": "delta", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "balance_after": { + "name": "balance_after", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "balance_change_reason", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "event_index": { + "name": "event_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "counterparty_account_id": { + "name": "counterparty_account_id", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "balance_changes_account_denom_height_idx": { + "name": "balance_changes_account_denom_height_idx", + "columns": [ + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "denom", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "balance_changes_height_event_index_idx": { + "name": "balance_changes_height_event_index_idx", + "columns": [ + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_index", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "balance_changes_account_id_accounts_id_fk": { + "name": "balance_changes_account_id_accounts_id_fk", + "tableFrom": "balance_changes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "balance_changes_counterparty_account_id_accounts_id_fk": { + "name": "balance_changes_counterparty_account_id_accounts_id_fk", + "tableFrom": "balance_changes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "counterparty_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.bids": { + "name": "bids", + "schema": "akash", + "columns": { + "deployment_id": { + "name": "deployment_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gseq": { + "name": "gseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "oseq": { + "name": "oseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "bseq": { + "name": "bseq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "provider_account_id": { + "name": "provider_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "price": { + "name": "price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "bid_state", + "typeSchema": "akash", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "created_height": { + "name": "created_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "closed_height": { + "name": "closed_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "bids_deployment_id_deployments_id_fk": { + "name": "bids_deployment_id_deployments_id_fk", + "tableFrom": "bids", + "tableTo": "deployments", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "bids_provider_account_id_accounts_id_fk": { + "name": "bids_provider_account_id_accounts_id_fk", + "tableFrom": "bids", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "provider_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "bids_deployment_id_gseq_oseq_bseq_provider_account_id_pk": { + "name": "bids_deployment_id_gseq_oseq_bseq_provider_account_id_pk", + "columns": [ + "deployment_id", + "gseq", + "oseq", + "bseq", + "provider_account_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.blocks": { + "name": "blocks", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "datetime": { + "name": "datetime", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "parent_hash": { + "name": "parent_hash", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "proposer_address": { + "name": "proposer_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tx_count": { + "name": "tx_count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.bme_canceled_records": { + "name": "bme_canceled_records", + "schema": "akash", + "columns": { + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "to_denom": { + "name": "to_denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "record_height": { + "name": "record_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "sequence": { + "name": "sequence", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cancel_reason": { + "name": "cancel_reason", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "owner_account_id": { + "name": "owner_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "to_account_id": { + "name": "to_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "coins_to_burn_denom": { + "name": "coins_to_burn_denom", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "coins_to_burn_amount": { + "name": "coins_to_burn_amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "denom_to_mint": { + "name": "denom_to_mint", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "bme_canceled_records_height_idx": { + "name": "bme_canceled_records_height_idx", + "columns": [ + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "bme_canceled_records_owner_account_id_accounts_id_fk": { + "name": "bme_canceled_records_owner_account_id_accounts_id_fk", + "tableFrom": "bme_canceled_records", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "owner_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "bme_canceled_records_to_account_id_accounts_id_fk": { + "name": "bme_canceled_records_to_account_id_accounts_id_fk", + "tableFrom": "bme_canceled_records", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "to_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "bme_canceled_records_record_height_sequence_denom_to_denom_source_pk": { + "name": "bme_canceled_records_record_height_sequence_denom_to_denom_source_pk", + "columns": [ + "record_height", + "sequence", + "denom", + "to_denom", + "source" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.bme_ledger_records": { + "name": "bme_ledger_records", + "schema": "akash", + "columns": { + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "to_denom": { + "name": "to_denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "record_height": { + "name": "record_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "sequence": { + "name": "sequence", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "burned_from_account_id": { + "name": "burned_from_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "minted_to_account_id": { + "name": "minted_to_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "burned_denom": { + "name": "burned_denom", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "burned_amount": { + "name": "burned_amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "burned_price": { + "name": "burned_price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": false + }, + "minted_denom": { + "name": "minted_denom", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "minted_amount": { + "name": "minted_amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "minted_price": { + "name": "minted_price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": false + }, + "spread_denom": { + "name": "spread_denom", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "spread_amount": { + "name": "spread_amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "remint_credit_issued_amount": { + "name": "remint_credit_issued_amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "remint_credit_accrued_amount": { + "name": "remint_credit_accrued_amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "bme_ledger_records_height_idx": { + "name": "bme_ledger_records_height_idx", + "columns": [ + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "bme_ledger_records_burned_denom_height_idx": { + "name": "bme_ledger_records_burned_denom_height_idx", + "columns": [ + { + "expression": "burned_denom", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "bme_ledger_records_minted_denom_height_idx": { + "name": "bme_ledger_records_minted_denom_height_idx", + "columns": [ + { + "expression": "minted_denom", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "bme_ledger_records_burned_from_account_id_accounts_id_fk": { + "name": "bme_ledger_records_burned_from_account_id_accounts_id_fk", + "tableFrom": "bme_ledger_records", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "burned_from_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "bme_ledger_records_minted_to_account_id_accounts_id_fk": { + "name": "bme_ledger_records_minted_to_account_id_accounts_id_fk", + "tableFrom": "bme_ledger_records", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "minted_to_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "bme_ledger_records_record_height_sequence_denom_to_denom_source_pk": { + "name": "bme_ledger_records_record_height_sequence_denom_to_denom_source_pk", + "columns": [ + "record_height", + "sequence", + "denom", + "to_denom", + "source" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.bme_status_changes": { + "name": "bme_status_changes", + "schema": "akash", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ordinal": { + "name": "ordinal", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "previous_status": { + "name": "previous_status", + "type": "bme_mint_status", + "typeSchema": "akash", + "primaryKey": false, + "notNull": true + }, + "new_status": { + "name": "new_status", + "type": "bme_mint_status", + "typeSchema": "akash", + "primaryKey": false, + "notNull": true + }, + "collateral_ratio": { + "name": "collateral_ratio", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "bme_status_changes_height_ordinal_pk": { + "name": "bme_status_changes_height_ordinal_pk", + "columns": [ + "height", + "ordinal" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.delegations": { + "name": "delegations", + "schema": "cosmos", + "columns": { + "delegator_account_id": { + "name": "delegator_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "validator_operator_address": { + "name": "validator_operator_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "shares": { + "name": "shares", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "delegations_delegator_account_id_accounts_id_fk": { + "name": "delegations_delegator_account_id_accounts_id_fk", + "tableFrom": "delegations", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "delegator_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "delegations_delegator_account_id_validator_operator_address_pk": { + "name": "delegations_delegator_account_id_validator_operator_address_pk", + "columns": [ + "delegator_account_id", + "validator_operator_address" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.deployment_events": { + "name": "deployment_events", + "schema": "akash", + "columns": { + "deployment_id": { + "name": "deployment_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ordinal": { + "name": "ordinal", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "msg_index": { + "name": "msg_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "deployment_event_type", + "typeSchema": "akash", + "primaryKey": false, + "notNull": true + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "deployment_events_deployment_id_deployments_id_fk": { + "name": "deployment_events_deployment_id_deployments_id_fk", + "tableFrom": "deployment_events", + "tableTo": "deployments", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "deployment_events_deployment_id_height_ordinal_pk": { + "name": "deployment_events_deployment_id_height_ordinal_pk", + "columns": [ + "deployment_id", + "height", + "ordinal" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.deployment_group_resources": { + "name": "deployment_group_resources", + "schema": "akash", + "columns": { + "deployment_group_id": { + "name": "deployment_group_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "idx": { + "name": "idx", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "cpu_units": { + "name": "cpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gpu_units": { + "name": "gpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gpu_vendor": { + "name": "gpu_vendor", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "gpu_model": { + "name": "gpu_model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "memory_bytes": { + "name": "memory_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ephemeral_storage_bytes": { + "name": "ephemeral_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "persistent_storage_bytes": { + "name": "persistent_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "price": { + "name": "price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "price_denom": { + "name": "price_denom", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "deployment_group_resources_deployment_group_id_deployment_groups_id_fk": { + "name": "deployment_group_resources_deployment_group_id_deployment_groups_id_fk", + "tableFrom": "deployment_group_resources", + "tableTo": "deployment_groups", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_group_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "deployment_group_resources_deployment_group_id_idx_pk": { + "name": "deployment_group_resources_deployment_group_id_idx_pk", + "columns": [ + "deployment_group_id", + "idx" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.deployment_groups": { + "name": "deployment_groups", + "schema": "akash", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "deployment_id": { + "name": "deployment_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gseq": { + "name": "gseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "group_state", + "typeSchema": "akash", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "closed_height": { + "name": "closed_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "deployment_groups_deployment_gseq_idx": { + "name": "deployment_groups_deployment_gseq_idx", + "columns": [ + { + "expression": "deployment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "gseq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "deployment_groups_deployment_id_deployments_id_fk": { + "name": "deployment_groups_deployment_id_deployments_id_fk", + "tableFrom": "deployment_groups", + "tableTo": "deployments", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.deployments": { + "name": "deployments", + "schema": "akash", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "owner_account_id": { + "name": "owner_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "dseq": { + "name": "dseq", + "type": "numeric(20, 0)", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deposit": { + "name": "deposit", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "balance": { + "name": "balance", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "withdrawn_amount": { + "name": "withdrawn_amount", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "block_rate": { + "name": "block_rate", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "last_withdraw_height": { + "name": "last_withdraw_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "last_processed_height": { + "name": "last_processed_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_height": { + "name": "created_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "closed_height": { + "name": "closed_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "closed_at": { + "name": "closed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "close_reason": { + "name": "close_reason", + "type": "deployment_close_reason", + "typeSchema": "akash", + "primaryKey": false, + "notNull": false + }, + "cpu_units": { + "name": "cpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gpu_units": { + "name": "gpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "memory_bytes": { + "name": "memory_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ephemeral_storage_bytes": { + "name": "ephemeral_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "persistent_storage_bytes": { + "name": "persistent_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "deployments_owner_dseq_idx": { + "name": "deployments_owner_dseq_idx", + "columns": [ + { + "expression": "owner_account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dseq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "deployments_owner_created_idx": { + "name": "deployments_owner_created_idx", + "columns": [ + { + "expression": "owner_account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "deployments_open_idx": { + "name": "deployments_open_idx", + "columns": [ + { + "expression": "created_height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"akash\".\"deployments\".\"closed_height\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "deployments_owner_account_id_accounts_id_fk": { + "name": "deployments_owner_account_id_accounts_id_fk", + "tableFrom": "deployments", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "owner_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.indexer_state": { + "name": "indexer_state", + "schema": "", + "columns": { + "stream": { + "name": "stream", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "last_height": { + "name": "last_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.leases": { + "name": "leases", + "schema": "akash", + "columns": { + "deployment_id": { + "name": "deployment_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "deployment_group_id": { + "name": "deployment_group_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gseq": { + "name": "gseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "oseq": { + "name": "oseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "bseq": { + "name": "bseq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "provider_account_id": { + "name": "provider_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "price": { + "name": "price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "balance": { + "name": "balance", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "withdrawn_amount": { + "name": "withdrawn_amount", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "predicted_closed_height": { + "name": "predicted_closed_height", + "type": "numeric(30, 0)", + "primaryKey": false, + "notNull": true + }, + "created_height": { + "name": "created_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "closed_height": { + "name": "closed_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "closed_at": { + "name": "closed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "cpu_units": { + "name": "cpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gpu_units": { + "name": "gpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "memory_bytes": { + "name": "memory_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ephemeral_storage_bytes": { + "name": "ephemeral_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "persistent_storage_bytes": { + "name": "persistent_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "leases_provider_idx": { + "name": "leases_provider_idx", + "columns": [ + { + "expression": "provider_account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "closed_height", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "leases_open_idx": { + "name": "leases_open_idx", + "columns": [ + { + "expression": "deployment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"akash\".\"leases\".\"closed_height\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "leases_deployment_id_deployments_id_fk": { + "name": "leases_deployment_id_deployments_id_fk", + "tableFrom": "leases", + "tableTo": "deployments", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "leases_deployment_group_id_deployment_groups_id_fk": { + "name": "leases_deployment_group_id_deployment_groups_id_fk", + "tableFrom": "leases", + "tableTo": "deployment_groups", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_group_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "leases_provider_account_id_accounts_id_fk": { + "name": "leases_provider_account_id_accounts_id_fk", + "tableFrom": "leases", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "provider_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "leases_deployment_id_gseq_oseq_bseq_provider_account_id_pk": { + "name": "leases_deployment_id_gseq_oseq_bseq_provider_account_id_pk", + "columns": [ + "deployment_id", + "gseq", + "oseq", + "bseq", + "provider_account_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.message_dead_letters": { + "name": "message_dead_letters", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type_id": { + "name": "type_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "raw": { + "name": "raw", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "message_dead_letters_type_id_idx": { + "name": "message_dead_letters_type_id_idx", + "columns": [ + { + "expression": "type_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "message_dead_letters_type_id_message_types_id_fk": { + "name": "message_dead_letters_type_id_message_types_id_fk", + "tableFrom": "message_dead_letters", + "tableTo": "message_types", + "schemaTo": "cosmos", + "columnsFrom": [ + "type_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "message_dead_letters_height_tx_index_index_pk": { + "name": "message_dead_letters_height_tx_index_index_pk", + "columns": [ + "height", + "tx_index", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.message_types": { + "name": "message_types", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "message_types_type_idx": { + "name": "message_types_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.messages": { + "name": "messages", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type_id": { + "name": "type_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "body": { + "name": "body", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "messages_type_id_idx": { + "name": "messages_type_id_idx", + "columns": [ + { + "expression": "type_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "messages_type_id_message_types_id_fk": { + "name": "messages_type_id_message_types_id_fk", + "tableFrom": "messages", + "tableTo": "message_types", + "schemaTo": "cosmos", + "columnsFrom": [ + "type_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messages_height_tx_index_index_pk": { + "name": "messages_height_tx_index_index_pk", + "columns": [ + "height", + "tx_index", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.proposal_deposits": { + "name": "proposal_deposits", + "schema": "cosmos", + "columns": { + "proposal_id": { + "name": "proposal_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "depositor_account_id": { + "name": "depositor_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "amount": { + "name": "amount", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "proposal_deposits_depositor_account_id_accounts_id_fk": { + "name": "proposal_deposits_depositor_account_id_accounts_id_fk", + "tableFrom": "proposal_deposits", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "depositor_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "proposal_deposits_proposal_id_depositor_account_id_height_pk": { + "name": "proposal_deposits_proposal_id_depositor_account_id_height_pk", + "columns": [ + "proposal_id", + "depositor_account_id", + "height" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.proposal_votes": { + "name": "proposal_votes", + "schema": "cosmos", + "columns": { + "proposal_id": { + "name": "proposal_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "voter_account_id": { + "name": "voter_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "options": { + "name": "options", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "proposal_votes_voter_account_id_accounts_id_fk": { + "name": "proposal_votes_voter_account_id_accounts_id_fk", + "tableFrom": "proposal_votes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "voter_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "proposal_votes_proposal_id_voter_account_id_pk": { + "name": "proposal_votes_proposal_id_voter_account_id_pk", + "columns": [ + "proposal_id", + "voter_account_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.proposals": { + "name": "proposals", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "proposer_account_id": { + "name": "proposer_account_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "messages": { + "name": "messages", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "proposal_status", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + }, + "submit_time": { + "name": "submit_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deposit_end_time": { + "name": "deposit_end_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "voting_start_time": { + "name": "voting_start_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "voting_end_time": { + "name": "voting_end_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "total_deposit": { + "name": "total_deposit", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "final_tally_yes": { + "name": "final_tally_yes", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "final_tally_abstain": { + "name": "final_tally_abstain", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "final_tally_no": { + "name": "final_tally_no", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "final_tally_no_with_veto": { + "name": "final_tally_no_with_veto", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "submit_height": { + "name": "submit_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "proposals_proposer_account_id_accounts_id_fk": { + "name": "proposals_proposer_account_id_accounts_id_fk", + "tableFrom": "proposals", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "proposer_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.provider_audit_signatures": { + "name": "provider_audit_signatures", + "schema": "akash", + "columns": { + "owner_account_id": { + "name": "owner_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "auditor_account_id": { + "name": "auditor_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "provider_audit_signatures_owner_account_id_accounts_id_fk": { + "name": "provider_audit_signatures_owner_account_id_accounts_id_fk", + "tableFrom": "provider_audit_signatures", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "owner_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "provider_audit_signatures_auditor_account_id_accounts_id_fk": { + "name": "provider_audit_signatures_auditor_account_id_accounts_id_fk", + "tableFrom": "provider_audit_signatures", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "auditor_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "provider_audit_signatures_owner_account_id_auditor_account_id_key_pk": { + "name": "provider_audit_signatures_owner_account_id_auditor_account_id_key_pk", + "columns": [ + "owner_account_id", + "auditor_account_id", + "key" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.providers": { + "name": "providers", + "schema": "akash", + "columns": { + "owner_account_id": { + "name": "owner_account_id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "host_uri": { + "name": "host_uri", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attributes": { + "name": "attributes", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "last_processed_height": { + "name": "last_processed_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_height": { + "name": "created_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "updated_height": { + "name": "updated_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "deleted_height": { + "name": "deleted_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "providers_owner_account_id_accounts_id_fk": { + "name": "providers_owner_account_id_accounts_id_fk", + "tableFrom": "providers", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "owner_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.transactions": { + "name": "transactions", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "code": { + "name": "code", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "gas_used": { + "name": "gas_used", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gas_wanted": { + "name": "gas_wanted", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "fee": { + "name": "fee", + "type": "jsonb", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "transactions_hash_idx": { + "name": "transactions_hash_idx", + "columns": [ + { + "expression": "hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "transactions_height_index_pk": { + "name": "transactions_height_index_pk", + "columns": [ + "height", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.unbonding_delegations": { + "name": "unbonding_delegations", + "schema": "cosmos", + "columns": { + "delegator_account_id": { + "name": "delegator_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "validator_operator_address": { + "name": "validator_operator_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "creation_height": { + "name": "creation_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "completion_time": { + "name": "completion_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "initial_balance": { + "name": "initial_balance", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "balance": { + "name": "balance", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "unbonding_delegations_delegator_account_id_accounts_id_fk": { + "name": "unbonding_delegations_delegator_account_id_accounts_id_fk", + "tableFrom": "unbonding_delegations", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "delegator_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "unbonding_delegations_delegator_account_id_validator_operator_address_creation_height_pk": { + "name": "unbonding_delegations_delegator_account_id_validator_operator_address_creation_height_pk", + "columns": [ + "delegator_account_id", + "validator_operator_address", + "creation_height" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.validators": { + "name": "validators", + "schema": "cosmos", + "columns": { + "operator_address": { + "name": "operator_address", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_address": { + "name": "account_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "hex_address": { + "name": "hex_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "moniker": { + "name": "moniker", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "identity": { + "name": "identity", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "details": { + "name": "details", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "security_contact": { + "name": "security_contact", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commission_rate": { + "name": "commission_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "commission_max_rate": { + "name": "commission_max_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "commission_max_change_rate": { + "name": "commission_max_change_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "min_self_delegation": { + "name": "min_self_delegation", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "jailed": { + "name": "jailed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "status": { + "name": "status", + "type": "validator_status", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": false + }, + "tokens": { + "name": "tokens", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "delegator_shares": { + "name": "delegator_shares", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": false + }, + "unbonding_height": { + "name": "unbonding_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "unbonding_time": { + "name": "unbonding_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "cosmos.account_tx_role": { + "name": "account_tx_role", + "schema": "cosmos", + "values": [ + "signer", + "sender", + "receiver" + ] + }, + "cosmos.balance_change_reason": { + "name": "balance_change_reason", + "schema": "cosmos", + "values": [ + "genesis", + "transfer", + "fee", + "reward", + "commission", + "slash", + "gov", + "ibc", + "escrow", + "bme", + "mint", + "burn", + "staking" + ] + }, + "akash.bid_state": { + "name": "bid_state", + "schema": "akash", + "values": [ + "open", + "active", + "closed" + ] + }, + "akash.bme_mint_status": { + "name": "bme_mint_status", + "schema": "akash", + "values": [ + "mint_status_unspecified", + "mint_status_healthy", + "mint_status_warning", + "mint_status_halt_cr", + "mint_status_halt_oracle" + ] + }, + "akash.deployment_close_reason": { + "name": "deployment_close_reason", + "schema": "akash", + "values": [ + "close_message", + "overdrawn", + "close_event" + ] + }, + "akash.deployment_event_type": { + "name": "deployment_event_type", + "schema": "akash", + "values": [ + "created", + "deposited", + "updated", + "closed", + "group_closed", + "group_paused", + "group_started", + "bid_created", + "bid_closed", + "lease_created", + "lease_closed", + "lease_withdrawn" + ] + }, + "akash.group_state": { + "name": "group_state", + "schema": "akash", + "values": [ + "open", + "paused", + "closed" + ] + }, + "cosmos.proposal_status": { + "name": "proposal_status", + "schema": "cosmos", + "values": [ + "deposit_period", + "voting_period", + "passed", + "rejected", + "failed" + ] + }, + "cosmos.validator_status": { + "name": "validator_status", + "schema": "cosmos", + "values": [ + "unbonded", + "unbonding", + "bonded" + ] + }, + "cosmos.vote_option": { + "name": "vote_option", + "schema": "cosmos", + "values": [ + "yes", + "abstain", + "no", + "no_with_veto" + ] + } + }, + "schemas": { + "akash": "akash", + "cosmos": "cosmos" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/chain-indexer/drizzle/meta/_journal.json b/apps/chain-indexer/drizzle/meta/_journal.json index dc53cc0966..19bebd6d6b 100644 --- a/apps/chain-indexer/drizzle/meta/_journal.json +++ b/apps/chain-indexer/drizzle/meta/_journal.json @@ -57,6 +57,13 @@ "when": 1786898959325, "tag": "0007_secret_overlord", "breakpoints": true + }, + { + "idx": 8, + "version": "7", + "when": 1786949585273, + "tag": "0008_good_galactus", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/chain-indexer/src/akash/akash-deriver.ts b/apps/chain-indexer/src/akash/akash-deriver.ts index d69315550c..64b67088da 100644 --- a/apps/chain-indexer/src/akash/akash-deriver.ts +++ b/apps/chain-indexer/src/akash/akash-deriver.ts @@ -1,5 +1,5 @@ import type { AkashBlockChanges, AkashChange, AkashChangeBody } from "@src/akash/akash-changes"; -import { asInteger, asRecord, asString } from "@src/akash/json"; +import { asInteger, asRecord, asString, parseJsonRecord } from "@src/akash/json"; import { normalizeAuditMessage } from "@src/akash/normalize-audit"; import { normalizeDeploymentMessage } from "@src/akash/normalize-deployment"; import { normalizeMarketMessage } from "@src/akash/normalize-market"; @@ -116,14 +116,14 @@ function legacyLeaseClosed(attributes: Record): AkashChangeBody } function typedDeploymentClosed(attributes: Record): AkashChangeBody | null { - const id = parseIdAttribute(attributes.id); + const id = parseJsonRecord(attributes.id); const owner = asString(id?.owner); const dseq = asUint64String(id?.dseq); return owner && dseq ? { kind: "deploymentClosedEvent", key: { owner, dseq } } : null; } function typedLeaseClosed(attributes: Record): AkashChangeBody | null { - const id = parseIdAttribute(attributes.id); + const id = parseJsonRecord(attributes.id); const owner = asString(id?.owner); const dseq = asUint64String(id?.dseq); const gseq = asInteger(id?.gseq); @@ -134,15 +134,3 @@ function typedLeaseClosed(attributes: Record): AkashChangeBody | } return { kind: "leaseClosedEvent", key: { owner, dseq }, gseq, oseq, bseq: asInteger(id?.bseq), provider }; } - -/** The typed events carry their id as a JSON string attribute. */ -function parseIdAttribute(raw: string | undefined): Record | null { - if (!raw) { - return null; - } - try { - return asRecord(JSON.parse(raw)); - } catch { - return null; - } -} diff --git a/apps/chain-indexer/src/akash/akash-writer.service.ts b/apps/chain-indexer/src/akash/akash-writer.service.ts index 605f4e699b..9ff7faf6d7 100644 --- a/apps/chain-indexer/src/akash/akash-writer.service.ts +++ b/apps/chain-indexer/src/akash/akash-writer.service.ts @@ -11,6 +11,7 @@ import { sumLeaseRate } from "@src/akash/settlement"; import { insertChunked } from "@src/db/insert-chunked"; import { Accounts, Bids, DeploymentEvents, DeploymentGroupResources, DeploymentGroups, Deployments, Leases } from "@src/db/schema"; import { sqlExcluded } from "@src/db/sql-excluded"; +import { requireAccountId } from "@src/pipeline/balance/account-interner.service"; import type { ChainTransaction } from "@src/providers/db.provider"; import { LoggerService } from "@src/providers/logging.provider"; @@ -80,7 +81,7 @@ export class AkashWriter { } return [...byKey.values()] - .map(key => ({ key, ownerAccountId: this.#requireId(accountIds, key.owner) })) + .map(key => ({ key, ownerAccountId: requireAccountId(accountIds, key.owner) })) .sort((a, b) => a.ownerAccountId - b.ownerAccountId || compareDseq(a.key.dseq, b.key.dseq)); } @@ -234,7 +235,7 @@ export class AkashWriter { deploymentIds: Map ): Promise { const rows = touched.map(state => ({ - ownerAccountId: this.#requireId(accountIds, state.key.owner), + ownerAccountId: requireAccountId(accountIds, state.key.owner), dseq: state.key.dseq, denom: state.denom, deposit: state.deposit.toString(), @@ -278,7 +279,7 @@ export class AkashWriter { const idByOwnerDseq = new Map(inserted.map(row => [ownerDseqKey(row.ownerAccountId, row.dseq), row.id])); for (const state of touched) { - const id = idByOwnerDseq.get(ownerDseqKey(this.#requireId(accountIds, state.key.owner), state.key.dseq)); + const id = idByOwnerDseq.get(ownerDseqKey(requireAccountId(accountIds, state.key.owner), state.key.dseq)); if (id !== undefined) { deploymentIds.set(stateKey(state.key), id); } @@ -288,9 +289,9 @@ export class AkashWriter { if (missing.length > 0) { const rowsForMissing = await this.#selectDeploymentsForUpdate( tx, - missing.map(state => ({ key: state.key, ownerAccountId: this.#requireId(accountIds, state.key.owner) })) + missing.map(state => ({ key: state.key, ownerAccountId: requireAccountId(accountIds, state.key.owner) })) ); - const keyByOwnerDseq = new Map(missing.map(state => [ownerDseqKey(this.#requireId(accountIds, state.key.owner), state.key.dseq), state.key])); + const keyByOwnerDseq = new Map(missing.map(state => [ownerDseqKey(requireAccountId(accountIds, state.key.owner), state.key.dseq), state.key])); for (const row of rowsForMissing) { const key = keyByOwnerDseq.get(ownerDseqKey(row.ownerAccountId, row.dseq)); if (key) { @@ -360,7 +361,7 @@ export class AkashWriter { gseq: bid.gseq, oseq: bid.oseq, bseq: bid.bseq, - providerAccountId: this.#requireId(accountIds, bid.provider), + providerAccountId: requireAccountId(accountIds, bid.provider), price: decToString(bid.price), denom: bid.denom, state: bid.state, @@ -402,7 +403,7 @@ export class AkashWriter { gseq: lease.gseq, oseq: lease.oseq, bseq: lease.bseq, - providerAccountId: this.#requireId(accountIds, lease.provider), + providerAccountId: requireAccountId(accountIds, lease.provider), price: decToString(lease.price), denom: lease.denom, balance: decToString(lease.balance), @@ -467,14 +468,6 @@ export class AkashWriter { } } - #requireId(accountIds: Map, address: string): number { - const id = accountIds.get(address); - if (id === undefined) { - throw new Error(`No interned account id for address ${address}`); - } - return id; - } - #requireAddress(addressesById: Map, accountId: number): string { const address = addressesById.get(accountId); if (address === undefined) { diff --git a/apps/chain-indexer/src/akash/json.ts b/apps/chain-indexer/src/akash/json.ts index 360239ed32..9cf67ff8d4 100644 --- a/apps/chain-indexer/src/akash/json.ts +++ b/apps/chain-indexer/src/akash/json.ts @@ -6,6 +6,18 @@ export function asString(value: unknown): string | null { return typeof value === "string" && value.length > 0 ? value : null; } +/** Typed proto events carry structured attributes (ids, coins) as JSON strings. */ +export function parseJsonRecord(raw: string | undefined): Record | null { + if (!raw) { + return null; + } + try { + return asRecord(JSON.parse(raw)); + } catch { + return null; + } +} + export function asInteger(value: unknown): number | null { if (typeof value === "number") { return Number.isSafeInteger(value) && value >= 0 ? value : null; diff --git a/apps/chain-indexer/src/akash/provider-writer.service.ts b/apps/chain-indexer/src/akash/provider-writer.service.ts index f38198f0e3..1848aabe74 100644 --- a/apps/chain-indexer/src/akash/provider-writer.service.ts +++ b/apps/chain-indexer/src/akash/provider-writer.service.ts @@ -7,6 +7,7 @@ import { isProviderAuditChange, isProviderChange, isProviderRegistryChange } fro import { INSERT_CHUNK_SIZE } from "@src/db/insert-chunk-size"; import { ProviderAuditSignatures, Providers } from "@src/db/schema"; import { sqlExcluded } from "@src/db/sql-excluded"; +import { requireAccountId } from "@src/pipeline/balance/account-interner.service"; import type { ChainTransaction } from "@src/providers/db.provider"; import { LoggerService } from "@src/providers/logging.provider"; @@ -125,7 +126,7 @@ export class ProviderWriter { if (isProviderAuditChange(change)) { continue; } - const ownerAccountId = this.#requireId(ownerIds, change.owner); + const ownerAccountId = requireAccountId(ownerIds, change.owner); if (skippedOwners.has(ownerAccountId)) { continue; } @@ -200,8 +201,8 @@ export class ProviderWriter { height: number, accountIds: Map ): Promise { - const ownerAccountId = this.#requireId(accountIds, change.owner); - const auditorAccountId = this.#requireId(accountIds, change.auditor); + const ownerAccountId = requireAccountId(accountIds, change.owner); + const auditorAccountId = requireAccountId(accountIds, change.auditor); const rows = dedupeByKeyLastWins(change.attributes).map(attribute => ({ ownerAccountId, auditorAccountId, @@ -230,8 +231,8 @@ export class ProviderWriter { accountIds: Map ): Promise { const identity = and( - eq(ProviderAuditSignatures.ownerAccountId, this.#requireId(accountIds, change.owner)), - eq(ProviderAuditSignatures.auditorAccountId, this.#requireId(accountIds, change.auditor)), + eq(ProviderAuditSignatures.ownerAccountId, requireAccountId(accountIds, change.owner)), + eq(ProviderAuditSignatures.auditorAccountId, requireAccountId(accountIds, change.auditor)), lte(ProviderAuditSignatures.height, height) ); await tx.delete(ProviderAuditSignatures).where(change.keys.length > 0 ? and(identity, inArray(ProviderAuditSignatures.key, change.keys)) : identity); @@ -242,7 +243,7 @@ export class ProviderWriter { for (const block of blocks) { for (const change of block.changes) { if (isProviderRegistryChange(change)) { - ownerIds.set(change.owner, this.#requireId(accountIds, change.owner)); + ownerIds.set(change.owner, requireAccountId(accountIds, change.owner)); } } } @@ -261,14 +262,6 @@ export class ProviderWriter { } this.#logger.warn({ event: "PROVIDER_ORPHAN_REFERENCE", count: warnings.length, samples: warnings.slice(0, 5) }); } - - #requireId(accountIds: Map, address: string): number { - const id = accountIds.get(address); - if (id === undefined) { - throw new Error(`No interned account id for address ${address}`); - } - return id; - } } /** diff --git a/apps/chain-indexer/src/bme/bme-deriver.spec.ts b/apps/chain-indexer/src/bme/bme-deriver.spec.ts new file mode 100644 index 0000000000..f6611ae2ff --- /dev/null +++ b/apps/chain-indexer/src/bme/bme-deriver.spec.ts @@ -0,0 +1,371 @@ +import { describe, expect, it } from "vitest"; + +import { collectBmeAddresses, deriveBmeChanges } from "@src/bme/bme-deriver"; +import type { DecodedBlock, DecodedEvent } from "@src/pipeline/decoded-block"; + +const BLOCK_TIME = new Date("2026-08-13T00:00:00Z"); + +const EXECUTED_EVENT_TYPE = "akash.bme.v1.EventLedgerRecordExecuted"; +const STATUS_CHANGE_EVENT_TYPE = "akash.bme.v1.EventMintStatusChange"; +const CANCELED_EVENT_TYPE = "akash.bme.v1.EventLedgerRecordCanceled"; + +describe("deriveBmeChanges", () => { + it("derives an executed mint record (uakt to uact) from a block event", () => { + const changes = deriveBmeChanges( + block({ + blockEvents: [ + event(EXECUTED_EVENT_TYPE, { + id: '{"denom":"uakt","to_denom":"uact","source":"bme","height":12000,"sequence":3}', + burned_from: '"akash1burner"', + minted_to: '"akash1minter"', + burned: '{"coin":{"denom":"uakt","amount":"1000000"},"price":"1.150000000000000000"}', + minted: '{"coin":{"denom":"uact","amount":"1150000"},"price":"1.000000000000000000"}', + spread: '{"denom":"uakt","amount":"25"}', + remint_credit_accrued: '{"coin":{"denom":"uakt","amount":"1000000"},"price":"1.150000000000000000"}' + }) + ] + }) + ); + + expect(changes.warnings).toEqual([]); + expect(changes.changes).toEqual([ + { + kind: "ledgerRecordExecuted", + id: { denom: "uakt", toDenom: "uact", source: "bme", recordHeight: 12000, sequence: 3 }, + burnedFrom: "akash1burner", + mintedTo: "akash1minter", + burned: { denom: "uakt", amount: "1000000", price: "1.150000000000000000" }, + minted: { denom: "uact", amount: "1150000", price: "1.000000000000000000" }, + spread: { denom: "uakt", amount: "25" }, + remintCreditIssued: null, + remintCreditAccrued: { denom: "uakt", amount: "1000000", price: "1.150000000000000000" }, + txIndex: null, + ordinal: 0 + } + ]); + }); + + it("treats JSON-null coin attributes as absent, matching the wire shape of a pure mint", () => { + const changes = deriveBmeChanges( + block({ + blockEvents: [ + event(EXECUTED_EVENT_TYPE, { + id: '{"denom":"uakt","to_denom":"uact","source":"akash1requester","height":"4844258","sequence":"1"}', + burned: "null", + burned_from: '"akash1requester"', + minted: '{"coin":{"denom":"uact","amount":"10529003"},"price":"1.000000000000000000"}', + minted_to: '"akash1requester"', + remint_credit_accrued: '{"coin":{"denom":"uakt","amount":"20000000"},"price":"0.526450156686029160"}', + remint_credit_issued: "null", + spread: '{"denom":"uact","amount":"26322"}' + }) + ] + }) + ); + + expect(changes.warnings).toEqual([]); + expect(changes.changes).toEqual([ + expect.objectContaining({ + kind: "ledgerRecordExecuted", + id: { denom: "uakt", toDenom: "uact", source: "akash1requester", recordHeight: 4844258, sequence: 1 }, + burned: null, + minted: { denom: "uact", amount: "10529003", price: "1.000000000000000000" }, + spread: { denom: "uact", amount: "26322" }, + remintCreditIssued: null, + remintCreditAccrued: { denom: "uakt", amount: "20000000", price: "0.526450156686029160" } + }) + ]); + }); + + it("derives an executed burn record (uact to uakt) with remint credit issued", () => { + const changes = deriveBmeChanges( + block({ + blockEvents: [ + event(EXECUTED_EVENT_TYPE, { + id: '{"denom":"uact","to_denom":"uakt","source":"bme","height":"12100","sequence":"4"}', + burned_from: '"akash1burner"', + minted_to: '"akash1minter"', + burned: '{"coin":{"denom":"uact","amount":"500000"},"price":"1.000000000000000000"}', + minted: '{"coin":{"denom":"uakt","amount":"434782"},"price":"1.150000000000000000"}', + remint_credit_issued: '{"coin":{"denom":"uakt","amount":"434782"},"price":"1.150000000000000000"}' + }) + ] + }) + ); + + expect(changes.warnings).toEqual([]); + expect(changes.changes).toEqual([ + expect.objectContaining({ + kind: "ledgerRecordExecuted", + id: { denom: "uact", toDenom: "uakt", source: "bme", recordHeight: 12100, sequence: 4 }, + burned: { denom: "uact", amount: "500000", price: "1.000000000000000000" }, + remintCreditIssued: { denom: "uakt", amount: "434782", price: "1.150000000000000000" }, + remintCreditAccrued: null, + spread: null + }) + ]); + }); + + it("passes IBC denoms and unquoted address attributes through raw", () => { + const changes = deriveBmeChanges( + block({ + blockEvents: [ + event(EXECUTED_EVENT_TYPE, { + id: '{"denom":"ibc/170C677610AC31DF0904FFE09CD3B5C657492170E7E52372E48756B71E56F2F1","to_denom":"uact","source":"bme","height":9000,"sequence":1}', + burned_from: "akash1burner", + minted_to: "akash1minter", + burned: '{"coin":{"denom":"ibc/170C677610AC31DF0904FFE09CD3B5C657492170E7E52372E48756B71E56F2F1","amount":"7"},"price":"1.000000000000000000"}' + }) + ] + }) + ); + + expect(changes.warnings).toEqual([]); + expect(changes.changes).toEqual([ + expect.objectContaining({ + id: expect.objectContaining({ denom: "ibc/170C677610AC31DF0904FFE09CD3B5C657492170E7E52372E48756B71E56F2F1" }), + burnedFrom: "akash1burner", + mintedTo: "akash1minter", + burned: expect.objectContaining({ amount: "7" }), + minted: null + }) + ]); + }); + + it("derives a mint status change with JSON-quoted enum names and Dec ratio", () => { + const changes = deriveBmeChanges( + block({ + blockEvents: [ + event(STATUS_CHANGE_EVENT_TYPE, { + previous_status: '"mint_status_healthy"', + new_status: '"mint_status_warning"', + collateral_ratio: '"1.750000000000000000"' + }) + ] + }) + ); + + expect(changes.warnings).toEqual([]); + expect(changes.changes).toEqual([ + { + kind: "mintStatusChange", + previousStatus: "mint_status_healthy", + newStatus: "mint_status_warning", + collateralRatio: "1.750000000000000000", + txIndex: null, + ordinal: 0 + } + ]); + }); + + it("rejects a status change with a value outside the known mint statuses", () => { + const changes = deriveBmeChanges( + block({ + blockEvents: [ + event(STATUS_CHANGE_EVENT_TYPE, { + previous_status: '"mint_status_healthy"', + new_status: '"mint_status_brand_new"', + collateral_ratio: '"1.75"' + }) + ] + }) + ); + + expect(changes.changes).toEqual([]); + expect(changes.warnings).toHaveLength(1); + expect(changes.warnings[0]).toContain(STATUS_CHANGE_EVENT_TYPE); + }); + + it("derives a canceled record", () => { + const changes = deriveBmeChanges( + block({ + blockEvents: [ + event(CANCELED_EVENT_TYPE, { + id: '{"denom":"uakt","to_denom":"uact","source":"bme","height":12000,"sequence":5}', + cancel_reason: '"insufficient_funds"', + owner: '"akash1owner"', + to: '"akash1dest"', + coins_to_burn: '{"denom":"uakt","amount":"42"}', + denom_to_mint: '"uact"' + }) + ] + }) + ); + + expect(changes.warnings).toEqual([]); + expect(changes.changes).toEqual([ + { + kind: "ledgerRecordCanceled", + id: { denom: "uakt", toDenom: "uact", source: "bme", recordHeight: 12000, sequence: 5 }, + cancelReason: "insufficient_funds", + owner: "akash1owner", + to: "akash1dest", + coinsToBurn: { denom: "uakt", amount: "42" }, + denomToMint: "uact", + txIndex: null, + ordinal: 0 + } + ]); + }); + + it("assigns ordinals across transaction events then block events in scan order", () => { + const changes = deriveBmeChanges( + block({ + txEvents: [event(STATUS_CHANGE_EVENT_TYPE, statusChangeAttributes())], + blockEvents: [ + event(EXECUTED_EVENT_TYPE, executedAttributes({ sequence: 1 })), + event(STATUS_CHANGE_EVENT_TYPE, statusChangeAttributes()), + event(EXECUTED_EVENT_TYPE, executedAttributes({ sequence: 2 })) + ] + }) + ); + + expect(changes.changes.map(change => [change.kind, change.txIndex, change.ordinal])).toEqual([ + ["mintStatusChange", 0, 0], + ["ledgerRecordExecuted", null, 1], + ["mintStatusChange", null, 2], + ["ledgerRecordExecuted", null, 3] + ]); + }); + + it("skips events of failed transactions but keeps block events", () => { + const changes = deriveBmeChanges( + block({ + code: 1, + txEvents: [event(STATUS_CHANGE_EVENT_TYPE, statusChangeAttributes())], + blockEvents: [event(EXECUTED_EVENT_TYPE, executedAttributes({ sequence: 1 }))] + }) + ); + + expect(changes.changes.map(change => [change.kind, change.ordinal])).toEqual([["ledgerRecordExecuted", 0]]); + }); + + it("drops an event with a malformed id, keeps its ordinal slot and derives the rest", () => { + const changes = deriveBmeChanges( + block({ + blockEvents: [ + event(EXECUTED_EVENT_TYPE, { ...executedAttributes({ sequence: 1 }), id: "not-json" }), + event(EXECUTED_EVENT_TYPE, executedAttributes({ sequence: 2 })) + ] + }) + ); + + expect(changes.warnings).toHaveLength(1); + expect(changes.warnings[0]).toContain(EXECUTED_EVENT_TYPE); + expect(changes.changes.map(change => [change.kind, change.ordinal])).toEqual([["ledgerRecordExecuted", 1]]); + }); + + it("drops an executed event missing a party address", () => { + const attributes = executedAttributes({ sequence: 1 }); + delete (attributes as Record).minted_to; + + const changes = deriveBmeChanges(block({ blockEvents: [event(EXECUTED_EVENT_TYPE, attributes)] })); + + expect(changes.changes).toEqual([]); + expect(changes.warnings).toHaveLength(1); + }); + + it("drops an executed event whose coin amount is not an integer string", () => { + const changes = deriveBmeChanges( + block({ + blockEvents: [ + event(EXECUTED_EVENT_TYPE, { + ...executedAttributes({ sequence: 1 }), + burned: '{"coin":{"denom":"uakt","amount":"1.5"},"price":"1.0"}' + }) + ] + }) + ); + + expect(changes.changes).toEqual([]); + expect(changes.warnings).toHaveLength(1); + }); + + it("ignores vault funded and unrelated events", () => { + const changes = deriveBmeChanges( + block({ + blockEvents: [ + event("akash.bme.v1.EventVaultFunded", { + amount: '{"denom":"uakt","amount":"100"}', + source: '"bme"', + new_vault_balance: '{"denom":"uakt","amount":"5000"}' + }), + event("transfer", { sender: "a", recipient: "b", amount: "1uakt" }) + ] + }) + ); + + expect(changes.changes).toEqual([]); + expect(changes.warnings).toEqual([]); + }); +}); + +describe("collectBmeAddresses", () => { + it("collects executed record parties and canceled record owner and destination", () => { + const executed = deriveBmeChanges(block({ blockEvents: [event(EXECUTED_EVENT_TYPE, executedAttributes({ sequence: 1 }))] })); + const canceled = deriveBmeChanges( + block({ + height: 101, + blockEvents: [ + event(CANCELED_EVENT_TYPE, { + id: '{"denom":"uakt","to_denom":"uact","source":"bme","height":12000,"sequence":5}', + cancel_reason: '"epsilon"', + owner: '"akash1owner"', + to: '"akash1dest"', + denom_to_mint: '"uact"' + }) + ] + }) + ); + + expect(collectBmeAddresses([executed, canceled])).toEqual(new Set(["akash1burner", "akash1minter", "akash1owner", "akash1dest"])); + }); +}); + +function executedAttributes(input: { sequence: number }): Record { + return { + id: `{"denom":"uakt","to_denom":"uact","source":"bme","height":12000,"sequence":${input.sequence}}`, + burned_from: '"akash1burner"', + minted_to: '"akash1minter"', + burned: '{"coin":{"denom":"uakt","amount":"1000000"},"price":"1.150000000000000000"}', + minted: '{"coin":{"denom":"uact","amount":"1150000"},"price":"1.000000000000000000"}' + }; +} + +function statusChangeAttributes(): Record { + return { + previous_status: '"mint_status_healthy"', + new_status: '"mint_status_warning"', + collateral_ratio: '"1.750000000000000000"' + }; +} + +function block(input: { height?: number; code?: number; txEvents?: DecodedEvent[]; blockEvents?: DecodedEvent[] }): DecodedBlock { + return { + height: input.height ?? 100, + datetime: BLOCK_TIME, + hash: Buffer.alloc(0), + parentHash: null, + proposerAddress: "P", + transactions: input.txEvents + ? [ + { + index: 0, + hash: Buffer.alloc(0), + code: input.code ?? 0, + gasUsed: 0, + gasWanted: 0, + fee: [], + messages: [], + events: input.txEvents, + signerAddresses: [] + } + ] + : [], + blockEvents: input.blockEvents ?? [] + }; +} + +function event(type: string, attributes: Record, msgIndex?: number): DecodedEvent { + return msgIndex === undefined ? { type, attributes } : { type, attributes, msgIndex }; +} diff --git a/apps/chain-indexer/src/bme/bme-deriver.ts b/apps/chain-indexer/src/bme/bme-deriver.ts new file mode 100644 index 0000000000..c8213d2263 --- /dev/null +++ b/apps/chain-indexer/src/bme/bme-deriver.ts @@ -0,0 +1,282 @@ +import { asInteger, asRecord, asString, parseJsonRecord } from "@src/akash/json"; +import { bmeMintStatus } from "@src/db/schema"; +import type { DecodedBlock, DecodedEvent } from "@src/pipeline/decoded-block"; + +const LEDGER_RECORD_EXECUTED_EVENT_TYPE = "akash.bme.v1.EventLedgerRecordExecuted"; +const MINT_STATUS_CHANGE_EVENT_TYPE = "akash.bme.v1.EventMintStatusChange"; +const LEDGER_RECORD_CANCELED_EVENT_TYPE = "akash.bme.v1.EventLedgerRecordCanceled"; + +const MINT_STATUSES = new Set(bmeMintStatus.enumValues); + +type BmeMintStatus = (typeof bmeMintStatus.enumValues)[number]; + +/** The on-chain LedgerRecordID; `recordHeight` is the record's creation height, not the execution block. */ +export interface BmeRecordId { + denom: string; + toDenom: string; + source: string; + recordHeight: number; + sequence: number; +} + +export interface BmeCoin { + denom: string; + amount: string; +} + +export interface BmeCoinPrice extends BmeCoin { + price: string | null; +} + +export type BmeChangeBody = + | { + kind: "ledgerRecordExecuted"; + id: BmeRecordId; + burnedFrom: string; + mintedTo: string; + burned: BmeCoinPrice | null; + minted: BmeCoinPrice | null; + spread: BmeCoin | null; + remintCreditIssued: BmeCoinPrice | null; + remintCreditAccrued: BmeCoinPrice | null; + } + | { kind: "mintStatusChange"; previousStatus: BmeMintStatus; newStatus: BmeMintStatus; collateralRatio: string } + | { + kind: "ledgerRecordCanceled"; + id: BmeRecordId; + cancelReason: string; + owner: string; + to: string; + coinsToBurn: BmeCoin | null; + denomToMint: string; + }; + +export type BmeChange = BmeChangeBody & { txIndex: number | null; ordinal: number }; + +export interface BmeBlockChanges { + height: number; + changes: BmeChange[]; + warnings: string[]; +} + +type ParsedChange = { change: BmeChangeBody } | { error: string }; + +/** + * Extracts the BME (burn-mint-equilibrium) lifecycle from a block's events: executed ledger records, + * mint status transitions and canceled records. Events of failed transactions are skipped; in practice + * BME fires in the EndBlocker, but tx events are scanned too since the natural keys dedupe either way. + * `ordinal` counts every BME-typed event in scan order — including ones that fail to parse — so a later + * parser fix replays with stable ordinals. Parse failures land in `warnings` for the writer to log; + * a malformed event must not halt the block. + */ +export function deriveBmeChanges(block: DecodedBlock): BmeBlockChanges { + const changes: BmeChange[] = []; + const warnings: string[] = []; + let ordinal = 0; + + const append = (events: DecodedEvent[], txIndex: number | null) => { + for (const event of events) { + const result = bmeChangeBody(event); + if (!result) { + continue; + } + if ("error" in result) { + warnings.push(`height=${block.height} ordinal=${ordinal} type=${event.type}: ${result.error}`); + } else { + changes.push({ ...result.change, txIndex, ordinal }); + } + ordinal += 1; + } + }; + + for (const tx of block.transactions) { + if (tx.code !== 0) { + continue; + } + append(tx.events, tx.index); + } + append(block.blockEvents, null); + + return { height: block.height, changes, warnings }; +} + +export function collectBmeAddresses(blocks: BmeBlockChanges[]): Set { + const addresses = new Set(); + for (const block of blocks) { + for (const change of block.changes) { + if (change.kind === "ledgerRecordExecuted") { + addresses.add(change.burnedFrom); + addresses.add(change.mintedTo); + } else if (change.kind === "ledgerRecordCanceled") { + addresses.add(change.owner); + addresses.add(change.to); + } + } + } + return addresses; +} + +function bmeChangeBody(event: DecodedEvent): ParsedChange | null { + switch (event.type) { + case LEDGER_RECORD_EXECUTED_EVENT_TYPE: + return ledgerRecordExecuted(event.attributes); + case MINT_STATUS_CHANGE_EVENT_TYPE: + return mintStatusChange(event.attributes); + case LEDGER_RECORD_CANCELED_EVENT_TYPE: + return ledgerRecordCanceled(event.attributes); + default: + return null; + } +} + +function ledgerRecordExecuted(attributes: Record): ParsedChange { + const id = parseRecordId(attributes.id); + if (!id) { + return { error: `unparseable id attribute: ${attributes.id}` }; + } + const burnedFrom = parseQuotedString(attributes.burned_from); + const mintedTo = parseQuotedString(attributes.minted_to); + if (!burnedFrom || !mintedTo) { + return { error: "missing burned_from or minted_to" }; + } + const burned = parseCoinPrice(attributes.burned); + const minted = parseCoinPrice(attributes.minted); + const spread = parseCoin(attributes.spread); + const remintCreditIssued = parseCoinPrice(attributes.remint_credit_issued); + const remintCreditAccrued = parseCoinPrice(attributes.remint_credit_accrued); + const malformed = [burned, minted, spread, remintCreditIssued, remintCreditAccrued].some(coin => coin === undefined); + if (malformed) { + return { error: "malformed coin attribute" }; + } + return { + change: { + kind: "ledgerRecordExecuted", + id, + burnedFrom, + mintedTo, + burned: burned ?? null, + minted: minted ?? null, + spread: spread ?? null, + remintCreditIssued: remintCreditIssued ?? null, + remintCreditAccrued: remintCreditAccrued ?? null + } + }; +} + +function mintStatusChange(attributes: Record): ParsedChange { + const previousStatus = parseQuotedString(attributes.previous_status); + const newStatus = parseQuotedString(attributes.new_status); + const collateralRatio = parseQuotedString(attributes.collateral_ratio); + if (!isMintStatus(previousStatus) || !isMintStatus(newStatus)) { + return { error: `unknown mint status: ${previousStatus} -> ${newStatus}` }; + } + if (!collateralRatio || !isDecString(collateralRatio)) { + return { error: `unparseable collateral_ratio: ${attributes.collateral_ratio}` }; + } + return { change: { kind: "mintStatusChange", previousStatus, newStatus, collateralRatio } }; +} + +function ledgerRecordCanceled(attributes: Record): ParsedChange { + const id = parseRecordId(attributes.id); + if (!id) { + return { error: `unparseable id attribute: ${attributes.id}` }; + } + const cancelReason = parseQuotedString(attributes.cancel_reason); + const owner = parseQuotedString(attributes.owner); + const to = parseQuotedString(attributes.to); + const denomToMint = parseQuotedString(attributes.denom_to_mint); + if (!cancelReason || !owner || !to || !denomToMint) { + return { error: "missing cancel_reason, owner, to or denom_to_mint" }; + } + const coinsToBurn = parseCoin(attributes.coins_to_burn); + if (coinsToBurn === undefined) { + return { error: "malformed coins_to_burn attribute" }; + } + return { change: { kind: "ledgerRecordCanceled", id, cancelReason, owner, to, coinsToBurn: coinsToBurn ?? null, denomToMint } }; +} + +function parseRecordId(raw: string | undefined): BmeRecordId | null { + const record = parseJsonRecord(raw); + const denom = asString(record?.denom); + const toDenom = asString(record?.to_denom); + const source = asString(record?.source); + const recordHeight = asInteger(record?.height); + const sequence = asInteger(record?.sequence); + if (!denom || !toDenom || !source || recordHeight === null || sequence === null) { + return null; + } + return { denom, toDenom, source, recordHeight, sequence }; +} + +/** + * `undefined` = the attribute is present but malformed; `null` = absent, which is a valid state. + * An unset proto message field reaches the wire as the literal JSON `null` (e.g. `burned` on a pure + * mint), so that counts as absent too. + */ +function parseCoinPrice(raw: string | undefined): BmeCoinPrice | null | undefined { + const parsed = parseJsonAttribute(raw); + if (parsed === null) { + return null; + } + const record = asRecord(parsed); + const coin = coinOf(asRecord(record?.coin)); + if (!coin) { + return undefined; + } + return { ...coin, price: asString(record?.price) }; +} + +function parseCoin(raw: string | undefined): BmeCoin | null | undefined { + const parsed = parseJsonAttribute(raw); + if (parsed === null) { + return null; + } + return coinOf(asRecord(parsed)) ?? undefined; +} + +/** `null` = absent (missing attribute or JSON null); `undefined` = present but not valid JSON. */ +function parseJsonAttribute(raw: string | undefined): unknown { + if (raw === undefined) { + return null; + } + try { + return JSON.parse(raw) ?? null; + } catch { + return undefined; + } +} + +function coinOf(record: Record | null): BmeCoin | null { + const denom = asString(record?.denom); + const amount = asString(record?.amount); + if (!denom || !amount || !/^\d+$/.test(amount)) { + return null; + } + return { denom, amount }; +} + +/** + * Strips the JSON string encoding CometBFT ABCI 2.0+ applies to proto string/Dec/enum attributes + * (e.g. `"bme"` or `"1.75"`), falling back to the raw value for unquoted legacy attributes. + */ +function parseQuotedString(raw: string | undefined): string | null { + if (!raw) { + return null; + } + if (raw.startsWith('"') && raw.endsWith('"')) { + try { + return asString(JSON.parse(raw)); + } catch { + return raw; + } + } + return raw; +} + +function isMintStatus(value: string | null): value is BmeMintStatus { + return value !== null && MINT_STATUSES.has(value); +} + +function isDecString(value: string): boolean { + return /^\d+(\.\d+)?$/.test(value); +} diff --git a/apps/chain-indexer/src/bme/bme-writer.service.spec.ts b/apps/chain-indexer/src/bme/bme-writer.service.spec.ts new file mode 100644 index 0000000000..b00651c3c7 --- /dev/null +++ b/apps/chain-indexer/src/bme/bme-writer.service.spec.ts @@ -0,0 +1,241 @@ +import { describe, expect, it } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import type { BmeBlockChanges, BmeChange } from "@src/bme/bme-deriver"; +import { BmeWriter } from "@src/bme/bme-writer.service"; +import { BmeCanceledRecords, BmeLedgerRecords, BmeStatusChanges } from "@src/db/schema"; +import type { LoggerService } from "@src/providers/logging.provider"; + +import { buildTxFake, rowsFor } from "@test/fakes/build-tx-fake"; + +const BURNER = "akash1burner"; +const MINTER = "akash1minter"; +const OWNER = "akash1owner"; +const DEST = "akash1dest"; +const ACCOUNT_IDS = new Map([ + [BURNER, 3], + [MINTER, 4], + [OWNER, 5], + [DEST, 6] +]); + +describe(BmeWriter.name, () => { + it("writes executed records with interned account ids and flattened coin prices", async () => { + const { writer, tx, inserts } = setup(); + + await writer.write( + tx, + [ + blockChanges(100, [ + { + kind: "ledgerRecordExecuted", + id: { denom: "uakt", toDenom: "uact", source: "bme", recordHeight: 90, sequence: 3 }, + burnedFrom: BURNER, + mintedTo: MINTER, + burned: { denom: "uakt", amount: "1000000", price: "1.150000000000000000" }, + minted: { denom: "uact", amount: "1150000", price: "1.000000000000000000" }, + spread: { denom: "uakt", amount: "25" }, + remintCreditIssued: null, + remintCreditAccrued: { denom: "uakt", amount: "1000000", price: "1.150000000000000000" }, + txIndex: null, + ordinal: 0 + } + ]) + ], + ACCOUNT_IDS + ); + + expect(rowsFor(inserts, BmeLedgerRecords)).toEqual([ + { + denom: "uakt", + toDenom: "uact", + source: "bme", + recordHeight: 90, + sequence: 3, + height: 100, + txIndex: null, + burnedFromAccountId: 3, + mintedToAccountId: 4, + burnedDenom: "uakt", + burnedAmount: "1000000", + burnedPrice: "1.150000000000000000", + mintedDenom: "uact", + mintedAmount: "1150000", + mintedPrice: "1.000000000000000000", + spreadDenom: "uakt", + spreadAmount: "25", + remintCreditIssuedAmount: null, + remintCreditAccruedAmount: "1000000" + } + ]); + }); + + it("defaults absent coin sides to zero amounts and null denoms", async () => { + const { writer, tx, inserts } = setup(); + + await writer.write( + tx, + [ + blockChanges(100, [ + { + kind: "ledgerRecordExecuted", + id: { denom: "uakt", toDenom: "uact", source: "bme", recordHeight: 100, sequence: 1 }, + burnedFrom: BURNER, + mintedTo: MINTER, + burned: null, + minted: null, + spread: null, + remintCreditIssued: null, + remintCreditAccrued: null, + txIndex: 2, + ordinal: 0 + } + ]) + ], + ACCOUNT_IDS + ); + + expect(rowsFor(inserts, BmeLedgerRecords)).toEqual([ + expect.objectContaining({ + txIndex: 2, + burnedDenom: null, + burnedAmount: "0", + burnedPrice: null, + mintedDenom: null, + mintedAmount: "0", + mintedPrice: null, + spreadDenom: null, + spreadAmount: null + }) + ]); + }); + + it("writes status changes and canceled records to their tables", async () => { + const { writer, tx, inserts } = setup(); + + await writer.write( + tx, + [ + blockChanges(200, [ + { + kind: "mintStatusChange", + previousStatus: "mint_status_healthy", + newStatus: "mint_status_halt_oracle", + collateralRatio: "0.900000000000000000", + txIndex: null, + ordinal: 0 + }, + { + kind: "ledgerRecordCanceled", + id: { denom: "uakt", toDenom: "uact", source: "bme", recordHeight: 195, sequence: 7 }, + cancelReason: "insufficient_funds", + owner: OWNER, + to: DEST, + coinsToBurn: { denom: "uakt", amount: "42" }, + denomToMint: "uact", + txIndex: null, + ordinal: 1 + } + ]) + ], + ACCOUNT_IDS + ); + + expect(rowsFor(inserts, BmeStatusChanges)).toEqual([ + { + height: 200, + ordinal: 0, + previousStatus: "mint_status_healthy", + newStatus: "mint_status_halt_oracle", + collateralRatio: "0.900000000000000000" + } + ]); + expect(rowsFor(inserts, BmeCanceledRecords)).toEqual([ + { + denom: "uakt", + toDenom: "uact", + source: "bme", + recordHeight: 195, + sequence: 7, + height: 200, + txIndex: null, + cancelReason: "insufficient_funds", + ownerAccountId: 5, + toAccountId: 6, + coinsToBurnDenom: "uakt", + coinsToBurnAmount: "42", + denomToMint: "uact" + } + ]); + }); + + it("does nothing for blocks without changes or warnings", async () => { + const { writer, tx, inserts, logger } = setup(); + + await writer.write(tx, [blockChanges(100, [])], ACCOUNT_IDS); + + expect(inserts).toEqual([]); + expect(logger.warn).not.toHaveBeenCalled(); + }); + + it("throws when a change references an address that was not interned", async () => { + const { writer, tx } = setup(); + + const write = writer.write( + tx, + [ + blockChanges(100, [ + { + kind: "ledgerRecordExecuted", + id: { denom: "uakt", toDenom: "uact", source: "bme", recordHeight: 100, sequence: 1 }, + burnedFrom: "akash1unknown", + mintedTo: MINTER, + burned: null, + minted: null, + spread: null, + remintCreditIssued: null, + remintCreditAccrued: null, + txIndex: null, + ordinal: 0 + } + ]) + ], + ACCOUNT_IDS + ); + + await expect(write).rejects.toThrow("akash1unknown"); + }); + + it("logs deriver warnings once with count and samples", async () => { + const { writer, tx, logger } = setup(); + + await writer.write( + tx, + [ + { height: 100, changes: [], warnings: ["height=100 ordinal=0 type=akash.bme.v1.EventLedgerRecordExecuted: unparseable id attribute: x"] }, + { height: 101, changes: [], warnings: ["height=101 ordinal=0 type=akash.bme.v1.EventMintStatusChange: unknown mint status: a -> b"] } + ], + ACCOUNT_IDS + ); + + expect(logger.warn).toHaveBeenCalledExactlyOnceWith({ + event: "BME_EVENT_PARSE_FAILED", + count: 2, + samples: [ + "height=100 ordinal=0 type=akash.bme.v1.EventLedgerRecordExecuted: unparseable id attribute: x", + "height=101 ordinal=0 type=akash.bme.v1.EventMintStatusChange: unknown mint status: a -> b" + ] + }); + }); + + function blockChanges(height: number, changes: BmeChange[]): BmeBlockChanges { + return { height, changes, warnings: [] }; + } + + function setup() { + const { tx, inserts } = buildTxFake(); + const logger = mock(); + const writer = new BmeWriter(logger); + return { writer, tx, inserts, logger }; + } +}); diff --git a/apps/chain-indexer/src/bme/bme-writer.service.ts b/apps/chain-indexer/src/bme/bme-writer.service.ts new file mode 100644 index 0000000000..83766fb339 --- /dev/null +++ b/apps/chain-indexer/src/bme/bme-writer.service.ts @@ -0,0 +1,111 @@ +import { inject, singleton } from "tsyringe"; + +import type { BmeBlockChanges, BmeChange } from "@src/bme/bme-deriver"; +import { insertChunked } from "@src/db/insert-chunked"; +import { BmeCanceledRecords, BmeLedgerRecords, BmeStatusChanges } from "@src/db/schema"; +import { requireAccountId } from "@src/pipeline/balance/account-interner.service"; +import type { ChainTransaction } from "@src/providers/db.provider"; +import { LoggerService } from "@src/providers/logging.provider"; + +type LedgerRecordRow = typeof BmeLedgerRecords.$inferInsert; +type StatusChangeRow = typeof BmeStatusChanges.$inferInsert; +type CanceledRecordRow = typeof BmeCanceledRecords.$inferInsert; + +/** + * Persists the BME lifecycle inside the block transaction. Rows are plain conflict-ignoring appends — + * ledger and canceled records are keyed by the on-chain LedgerRecordID and status changes by + * (height, ordinal), so replaying a block is a no-op without locks or watermarks. + */ +@singleton() +export class BmeWriter { + readonly #logger: LoggerService; + + constructor(@inject(LoggerService) logger: LoggerService) { + this.#logger = logger; + this.#logger.setContext("BME_WRITER"); + } + + async write(tx: ChainTransaction, blocks: BmeBlockChanges[], accountIds: Map): Promise { + this.#logWarnings(blocks); + + const ledgerRecords: LedgerRecordRow[] = []; + const statusChanges: StatusChangeRow[] = []; + const canceledRecords: CanceledRecordRow[] = []; + + for (const block of blocks) { + for (const change of block.changes) { + if (change.kind === "ledgerRecordExecuted") { + ledgerRecords.push(this.#ledgerRecordRow(block.height, change, accountIds)); + } else if (change.kind === "mintStatusChange") { + statusChanges.push(this.#statusChangeRow(block.height, change)); + } else { + canceledRecords.push(this.#canceledRecordRow(block.height, change, accountIds)); + } + } + } + + await insertChunked(tx, BmeLedgerRecords, ledgerRecords); + await insertChunked(tx, BmeStatusChanges, statusChanges); + await insertChunked(tx, BmeCanceledRecords, canceledRecords); + } + + #ledgerRecordRow(height: number, change: Extract, accountIds: Map): LedgerRecordRow { + return { + denom: change.id.denom, + toDenom: change.id.toDenom, + source: change.id.source, + recordHeight: change.id.recordHeight, + sequence: change.id.sequence, + height, + txIndex: change.txIndex, + burnedFromAccountId: requireAccountId(accountIds, change.burnedFrom), + mintedToAccountId: requireAccountId(accountIds, change.mintedTo), + burnedDenom: change.burned?.denom ?? null, + burnedAmount: change.burned?.amount ?? "0", + burnedPrice: change.burned?.price ?? null, + mintedDenom: change.minted?.denom ?? null, + mintedAmount: change.minted?.amount ?? "0", + mintedPrice: change.minted?.price ?? null, + spreadDenom: change.spread?.denom ?? null, + spreadAmount: change.spread?.amount ?? null, + remintCreditIssuedAmount: change.remintCreditIssued?.amount ?? null, + remintCreditAccruedAmount: change.remintCreditAccrued?.amount ?? null + }; + } + + #statusChangeRow(height: number, change: Extract): StatusChangeRow { + return { + height, + ordinal: change.ordinal, + previousStatus: change.previousStatus, + newStatus: change.newStatus, + collateralRatio: change.collateralRatio + }; + } + + #canceledRecordRow(height: number, change: Extract, accountIds: Map): CanceledRecordRow { + return { + denom: change.id.denom, + toDenom: change.id.toDenom, + source: change.id.source, + recordHeight: change.id.recordHeight, + sequence: change.id.sequence, + height, + txIndex: change.txIndex, + cancelReason: change.cancelReason, + ownerAccountId: requireAccountId(accountIds, change.owner), + toAccountId: requireAccountId(accountIds, change.to), + coinsToBurnDenom: change.coinsToBurn?.denom ?? null, + coinsToBurnAmount: change.coinsToBurn?.amount ?? null, + denomToMint: change.denomToMint + }; + } + + #logWarnings(blocks: BmeBlockChanges[]): void { + const warnings = blocks.flatMap(block => block.warnings); + if (warnings.length === 0) { + return; + } + this.#logger.warn({ event: "BME_EVENT_PARSE_FAILED", count: warnings.length, samples: warnings.slice(0, 5) }); + } +} diff --git a/apps/chain-indexer/src/db/schema.ts b/apps/chain-indexer/src/db/schema.ts index 99fcb2dd0b..815cf23e22 100644 --- a/apps/chain-indexer/src/db/schema.ts +++ b/apps/chain-indexer/src/db/schema.ts @@ -551,3 +551,97 @@ export const ProviderAuditSignatures = akashSchema.table( }, t => [primaryKey({ columns: [t.ownerAccountId, t.auditorAccountId, t.key] })] ); + +export const bmeMintStatus = akashSchema.enum("bme_mint_status", [ + "mint_status_unspecified", + "mint_status_healthy", + "mint_status_warning", + "mint_status_halt_cr", + "mint_status_halt_oracle" +]); + +/** + * One row per executed BME burn/mint, keyed by the full on-chain LedgerRecordID. `record_height` is + * the record's creation height from the id and can precede `height`, the block whose EndBlocker + * executed it (pending records execute later). Amounts are the exact u-denom integers from the + * event; prices keep the chain's 18-decimal Dec precision. Denoms are stored raw (uakt/uact/ibc/...). + */ +export const BmeLedgerRecords = akashSchema.table( + "bme_ledger_records", + { + denom: text("denom").notNull(), + toDenom: text("to_denom").notNull(), + source: text("source").notNull(), + recordHeight: bigint("record_height", { mode: "number" }).notNull(), + sequence: bigint("sequence", { mode: "number" }).notNull(), + height: bigint("height", { mode: "number" }).notNull(), + txIndex: integer("tx_index"), + burnedFromAccountId: integer("burned_from_account_id") + .notNull() + .references(() => Accounts.id), + mintedToAccountId: integer("minted_to_account_id") + .notNull() + .references(() => Accounts.id), + burnedDenom: text("burned_denom"), + burnedAmount: numeric("burned_amount", { precision: 38, scale: 0 }).notNull().default("0"), + burnedPrice: numeric("burned_price", { precision: 38, scale: 18 }), + mintedDenom: text("minted_denom"), + mintedAmount: numeric("minted_amount", { precision: 38, scale: 0 }).notNull().default("0"), + mintedPrice: numeric("minted_price", { precision: 38, scale: 18 }), + spreadDenom: text("spread_denom"), + spreadAmount: numeric("spread_amount", { precision: 38, scale: 0 }), + remintCreditIssuedAmount: numeric("remint_credit_issued_amount", { precision: 38, scale: 0 }), + remintCreditAccruedAmount: numeric("remint_credit_accrued_amount", { precision: 38, scale: 0 }) + }, + t => [ + primaryKey({ columns: [t.recordHeight, t.sequence, t.denom, t.toDenom, t.source] }), + index("bme_ledger_records_height_idx").on(t.height), + index("bme_ledger_records_burned_denom_height_idx").on(t.burnedDenom, t.height), + index("bme_ledger_records_minted_denom_height_idx").on(t.mintedDenom, t.height) + ] +); + +/** + * BME circuit-breaker transitions. `ordinal` is the event's deterministic position among the block's + * BME events, so re-committing a block conflicts instead of duplicating. + */ +export const BmeStatusChanges = akashSchema.table( + "bme_status_changes", + { + height: bigint("height", { mode: "number" }).notNull(), + ordinal: integer("ordinal").notNull(), + previousStatus: bmeMintStatus("previous_status").notNull(), + newStatus: bmeMintStatus("new_status").notNull(), + collateralRatio: numeric("collateral_ratio", { precision: 38, scale: 18 }).notNull() + }, + t => [primaryKey({ columns: [t.height, t.ordinal] })] +); + +/** + * Canceled BME ledger records, parsed from EventLedgerRecordCanceled — the legacy indexer kept these + * only as raw staging rows. Same natural key as executed records; a canceled record has no supply + * impact, so no amounts land in `bme_ledger_records`. + */ +export const BmeCanceledRecords = akashSchema.table( + "bme_canceled_records", + { + denom: text("denom").notNull(), + toDenom: text("to_denom").notNull(), + source: text("source").notNull(), + recordHeight: bigint("record_height", { mode: "number" }).notNull(), + sequence: bigint("sequence", { mode: "number" }).notNull(), + height: bigint("height", { mode: "number" }).notNull(), + txIndex: integer("tx_index"), + cancelReason: text("cancel_reason").notNull(), + ownerAccountId: integer("owner_account_id") + .notNull() + .references(() => Accounts.id), + toAccountId: integer("to_account_id") + .notNull() + .references(() => Accounts.id), + coinsToBurnDenom: text("coins_to_burn_denom"), + coinsToBurnAmount: numeric("coins_to_burn_amount", { precision: 38, scale: 0 }), + denomToMint: text("denom_to_mint").notNull() + }, + t => [primaryKey({ columns: [t.recordHeight, t.sequence, t.denom, t.toDenom, t.source] }), index("bme_canceled_records_height_idx").on(t.height)] +); diff --git a/apps/chain-indexer/src/pipeline/balance/account-interner.service.ts b/apps/chain-indexer/src/pipeline/balance/account-interner.service.ts index c252d7c5c4..a4554e27e0 100644 --- a/apps/chain-indexer/src/pipeline/balance/account-interner.service.ts +++ b/apps/chain-indexer/src/pipeline/balance/account-interner.service.ts @@ -15,6 +15,15 @@ import { CHAIN_DB } from "@src/providers/db.provider"; * It runs on the base connection, not the commit transaction, so the interned rows are visible when the * transaction inserts ledger rows that reference them. */ +/** A miss means the caller's address-collection pass and its row-building pass disagree — an indexer bug, not chain data. */ +export function requireAccountId(accountIds: Map, address: string): number { + const id = accountIds.get(address); + if (id === undefined) { + throw new Error(`No interned account id for address ${address}`); + } + return id; +} + @singleton() export class AccountInterner { readonly #db: ChainDatabase; diff --git a/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts b/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts index eb276ac896..d06b7409dd 100644 --- a/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts +++ b/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts @@ -5,6 +5,7 @@ import { mock } from "vitest-mock-extended"; import type { AkashWriter } from "@src/akash/akash-writer.service"; import type { ProviderWriter } from "@src/akash/provider-writer.service"; +import type { BmeWriter } from "@src/bme/bme-writer.service"; import { AccountTxs, Blocks, IndexerState, MessageDeadLetters, Messages, MessageTypes } from "@src/db/schema"; import type { GovWriter } from "@src/gov/gov-writer.service"; import type { AccountInterner } from "@src/pipeline/balance/account-interner.service"; @@ -268,6 +269,35 @@ describe(BlockCommitterService.name, () => { expect(providerWriter.write).toHaveBeenCalledWith(expect.anything(), akashWriter.write.mock.calls[0][1], akashWriter.write.mock.calls[0][2]); }); + + it("hands the bme writer the derived changes and interns the record parties", async () => { + const { committer, bmeWriter } = setup({ selectResults: [[{ id: 7, type: MSG_SEND }]] }); + + await committer.commit( + buildBlock([MSG_SEND], 10, { + events: [ + { + type: "akash.bme.v1.EventLedgerRecordExecuted", + attributes: { + id: '{"denom":"uakt","to_denom":"uact","source":"bme","height":9,"sequence":1}', + burned_from: '"akash1bmeburner"', + minted_to: '"akash1bmeminter"', + minted: '{"coin":{"denom":"uact","amount":"100"},"price":"1.000000000000000000"}' + } + } + ] + }) + ); + + expect(bmeWriter.write).toHaveBeenCalledWith( + expect.anything(), + [expect.objectContaining({ height: 10, changes: [expect.objectContaining({ kind: "ledgerRecordExecuted" })] })], + expect.any(Map) + ); + const accountIds = bmeWriter.write.mock.calls[0][2]; + expect(accountIds.get("akash1bmeburner")).toBeDefined(); + expect(accountIds.get("akash1bmeminter")).toBeDefined(); + }); }); function setup(input?: { @@ -321,9 +351,19 @@ describe(BlockCommitterService.name, () => { const govWriter = mock(); const akashWriter = mock(); const providerWriter = mock(); + const bmeWriter = mock(); const logger = mock(); - const committer = new BlockCommitterService(dbFake as unknown as ChainDatabase, interner, balanceWriter, govWriter, akashWriter, providerWriter, logger); - return { committer, insertedRows, conflictUpdates, deletions, interner, balanceWriter, govWriter, akashWriter, providerWriter, logger }; + const committer = new BlockCommitterService( + dbFake as unknown as ChainDatabase, + interner, + balanceWriter, + govWriter, + akashWriter, + providerWriter, + bmeWriter, + logger + ); + return { committer, insertedRows, conflictUpdates, deletions, interner, balanceWriter, govWriter, akashWriter, providerWriter, bmeWriter, logger }; } function buildBlock( diff --git a/apps/chain-indexer/src/pipeline/block-committer.service.ts b/apps/chain-indexer/src/pipeline/block-committer.service.ts index a1aec9c2b9..8d594c2e8e 100644 --- a/apps/chain-indexer/src/pipeline/block-committer.service.ts +++ b/apps/chain-indexer/src/pipeline/block-committer.service.ts @@ -7,12 +7,15 @@ import { collectAkashAddresses } from "@src/akash/akash-changes"; import { deriveAkashChanges } from "@src/akash/akash-deriver"; import { AkashWriter } from "@src/akash/akash-writer.service"; import { ProviderWriter } from "@src/akash/provider-writer.service"; +import type { BmeBlockChanges } from "@src/bme/bme-deriver"; +import { collectBmeAddresses, deriveBmeChanges } from "@src/bme/bme-deriver"; +import { BmeWriter } from "@src/bme/bme-writer.service"; import { INSERT_CHUNK_SIZE } from "@src/db/insert-chunk-size"; import { insertChunked } from "@src/db/insert-chunked"; import { AccountTxs, Blocks, IndexerState, MessageDeadLetters, Messages, MessageTypes, Transactions } from "@src/db/schema"; import { sqlExcluded } from "@src/db/sql-excluded"; import { GovWriter } from "@src/gov/gov-writer.service"; -import { AccountInterner } from "@src/pipeline/balance/account-interner.service"; +import { AccountInterner, requireAccountId } from "@src/pipeline/balance/account-interner.service"; import type { DerivedAccountTx } from "@src/pipeline/balance/account-tx-deriver"; import { deriveAccountTxs } from "@src/pipeline/balance/account-tx-deriver"; import type { DerivedBalanceChange } from "@src/pipeline/balance/balance-deriver"; @@ -49,6 +52,7 @@ export class BlockCommitterService { readonly #govWriter: GovWriter; readonly #akashWriter: AkashWriter; readonly #providerWriter: ProviderWriter; + readonly #bmeWriter: BmeWriter; readonly #logger: LoggerService; readonly #moduleRegistry = buildModuleAddressRegistry(); readonly #typeIds = new Map(); @@ -60,6 +64,7 @@ export class BlockCommitterService { @inject(GovWriter) govWriter: GovWriter, @inject(AkashWriter) akashWriter: AkashWriter, @inject(ProviderWriter) providerWriter: ProviderWriter, + @inject(BmeWriter) bmeWriter: BmeWriter, @inject(LoggerService) logger: LoggerService ) { this.#db = db; @@ -68,6 +73,7 @@ export class BlockCommitterService { this.#govWriter = govWriter; this.#akashWriter = akashWriter; this.#providerWriter = providerWriter; + this.#bmeWriter = bmeWriter; this.#logger = logger; this.#logger.setContext("COMMITTER"); } @@ -136,7 +142,8 @@ export class BlockCommitterService { const balanceChanges = blocks.flatMap(block => deriveBalanceChanges(block, this.#moduleRegistry)); const accountTxs = blocks.flatMap(block => deriveAccountTxs(block)); const akashChanges = blocks.map(block => deriveAkashChanges(block)); - const accountIds = await this.#internAccounts(balanceChanges, accountTxs, akashChanges); + const bmeChanges = blocks.map(block => deriveBmeChanges(block)); + const accountIds = await this.#internAccounts(balanceChanges, accountTxs, akashChanges, bmeChanges); const balanceIntents = this.#resolveBalanceChanges(balanceChanges, accountIds); const accountTxRows = this.#resolveAccountTxs(accountTxs, accountIds); @@ -153,6 +160,7 @@ export class BlockCommitterService { await this.#govWriter.writeForBlocks(tx, blocks, accountIds); await this.#akashWriter.write(tx, akashChanges, accountIds); await this.#providerWriter.write(tx, akashChanges, accountIds); + await this.#bmeWriter.write(tx, bmeChanges, accountIds); await tx .insert(IndexerState) @@ -257,13 +265,15 @@ export class BlockCommitterService { /** * Interns every address the batch touches — spenders, receivers, correlated counterparties, tx signers, - * and the deployment owners/providers/depositors of the akash changes — on the base connection before - * the commit transaction, so the derived rows can reference their account ids by foreign key. + * the deployment owners/providers/depositors of the akash changes and the parties of the bme changes — + * on the base connection before the commit transaction, so the derived rows can reference their + * account ids by foreign key. */ async #internAccounts( balanceChanges: DerivedBalanceChange[], accountTxs: DerivedAccountTx[], - akashChanges: AkashBlockChanges[] + akashChanges: AkashBlockChanges[], + bmeChanges: BmeBlockChanges[] ): Promise> { const addresses = new Set(); @@ -279,13 +289,16 @@ export class BlockCommitterService { for (const address of collectAkashAddresses(akashChanges)) { addresses.add(address); } + for (const address of collectBmeAddresses(bmeChanges)) { + addresses.add(address); + } return this.#interner.resolve(addresses); } #resolveBalanceChanges(changes: DerivedBalanceChange[], accountIds: Map): ResolvedBalanceChange[] { return changes.map(change => ({ - accountId: this.#requireId(accountIds, change.address), + accountId: requireAccountId(accountIds, change.address), counterpartyAccountId: change.counterpartyAddress ? accountIds.get(change.counterpartyAddress) ?? null : null, denom: change.denom, delta: change.delta, @@ -297,15 +310,7 @@ export class BlockCommitterService { } #resolveAccountTxs(rows: DerivedAccountTx[], accountIds: Map): (typeof AccountTxs.$inferInsert)[] { - return rows.map(row => ({ accountId: this.#requireId(accountIds, row.address), height: row.height, txIndex: row.txIndex, role: row.role })); - } - - #requireId(accountIds: Map, address: string): number { - const accountId = accountIds.get(address); - if (accountId === undefined) { - throw new Error(`No interned account id for address ${address}`); - } - return accountId; + return rows.map(row => ({ accountId: requireAccountId(accountIds, row.address), height: row.height, txIndex: row.txIndex, role: row.role })); } async #internMessageTypes(blocks: DecodedBlock[]): Promise> { diff --git a/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts b/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts index ce62247cae..0437e2d278 100644 --- a/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts +++ b/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts @@ -234,6 +234,28 @@ describe(BlockDecoderService.name, () => { expect(decoded.blockEvents).toEqual([{ type: "coinbase", attributes: { minter: "akash1mint", amount: "10uakt" } }]); }); + it("captures bme lifecycle block events for the bme handler but drops vault funded ones", () => { + const { decoder } = setup(); + + const decoded = decoder.decode( + buildBlock({ txs: [] }), + buildBlockResults([], { + finalize_block_events: [ + event("akash.bme.v1.EventLedgerRecordExecuted", { id: '{"denom":"uakt","to_denom":"uact","source":"bme","height":100,"sequence":1}' }), + event("akash.bme.v1.EventMintStatusChange", { previous_status: '"mint_status_healthy"', new_status: '"mint_status_warning"' }), + event("akash.bme.v1.EventLedgerRecordCanceled", { cancel_reason: '"epsilon"' }), + event("akash.bme.v1.EventVaultFunded", { source: '"bme"' }) + ] + }) + ); + + expect(decoded.blockEvents.map(e => e.type)).toEqual([ + "akash.bme.v1.EventLedgerRecordExecuted", + "akash.bme.v1.EventMintStatusChange", + "akash.bme.v1.EventLedgerRecordCanceled" + ]); + }); + it("falls back to begin and end block events when finalize is absent", () => { const { decoder } = setup(); diff --git a/apps/chain-indexer/src/pipeline/block-decoder.service.ts b/apps/chain-indexer/src/pipeline/block-decoder.service.ts index 08bade8bb2..2561ee8370 100644 --- a/apps/chain-indexer/src/pipeline/block-decoder.service.ts +++ b/apps/chain-indexer/src/pipeline/block-decoder.service.ts @@ -19,8 +19,10 @@ import type { RpcBlockResult, RpcBlockResultsResult, RpcEvent, RpcTxResult } fro * The ledger derives balances and reasons from the coin/transfer/mint/burn/slash events; the gov events carry * proposal ids and lifecycle transitions for the governance handler; the akash events catch deployment and * lease closes that happen as side effects of other messages (group close, overdraw) — `akash.v1` is the - * legacy string-attribute event of early mainnet, the typed events are the current chain's. Capturing the - * rest would waste memory across backfill batches. + * legacy string-attribute event of early mainnet, the typed events are the current chain's. The bme events + * carry the executed/canceled ledger records and mint status transitions the BME handler derives from + * (EventVaultFunded is deliberately absent: the vault's balance is already exact in the balance ledger). + * Capturing the rest would waste memory across backfill batches. */ const RELEVANT_EVENT_TYPES = new Set([ "coin_spent", @@ -34,7 +36,10 @@ const RELEVANT_EVENT_TYPES = new Set([ "inactive_proposal", "akash.v1", "akash.deployment.v1.EventDeploymentClosed", - "akash.market.v1.EventLeaseClosed" + "akash.market.v1.EventLeaseClosed", + "akash.bme.v1.EventLedgerRecordExecuted", + "akash.bme.v1.EventMintStatusChange", + "akash.bme.v1.EventLedgerRecordCanceled" ]); const MSG_INDEX_ATTRIBUTE = "msg_index"; From 8464895cac6dc0b00606b7bac81a16ae125351c9 Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Tue, 18 Aug 2026 11:09:49 +0530 Subject: [PATCH 12/14] feat(indexer): incremental network state and daily rollup aggregates (#3604) --- .../drizzle/0009_loud_blue_marvel.sql | 49 + .../drizzle/meta/0009_snapshot.json | 3057 +++++++++++++++++ apps/chain-indexer/drizzle/meta/_journal.json | 7 + apps/chain-indexer/package.json | 1 + .../src/akash/akash-writer.service.spec.ts | 17 +- .../src/akash/akash-writer.service.ts | 23 +- .../src/akash/network-delta.spec.ts | 198 ++ apps/chain-indexer/src/akash/network-delta.ts | 146 + apps/chain-indexer/src/db/schema.ts | 78 + .../src/network/day-close-usd.service.spec.ts | 85 + .../src/network/day-close-usd.service.ts | 50 + .../network-stats-writer.service.spec.ts | 305 ++ .../network/network-stats-writer.service.ts | 208 ++ .../src/network/recompute-usd.ts | 37 + .../pipeline/block-committer.service.spec.ts | 31 +- .../src/pipeline/block-committer.service.ts | 7 +- .../network-stats-reconciler.spec.ts | 207 ++ .../src/reconcile/network-stats-reconciler.ts | 174 + apps/chain-indexer/src/reconcile/reconcile.ts | 9 +- apps/chain-indexer/tsup.config.ts | 1 + 20 files changed, 4676 insertions(+), 14 deletions(-) create mode 100644 apps/chain-indexer/drizzle/0009_loud_blue_marvel.sql create mode 100644 apps/chain-indexer/drizzle/meta/0009_snapshot.json create mode 100644 apps/chain-indexer/src/akash/network-delta.spec.ts create mode 100644 apps/chain-indexer/src/akash/network-delta.ts create mode 100644 apps/chain-indexer/src/network/day-close-usd.service.spec.ts create mode 100644 apps/chain-indexer/src/network/day-close-usd.service.ts create mode 100644 apps/chain-indexer/src/network/network-stats-writer.service.spec.ts create mode 100644 apps/chain-indexer/src/network/network-stats-writer.service.ts create mode 100644 apps/chain-indexer/src/network/recompute-usd.ts create mode 100644 apps/chain-indexer/src/reconcile/network-stats-reconciler.spec.ts create mode 100644 apps/chain-indexer/src/reconcile/network-stats-reconciler.ts diff --git a/apps/chain-indexer/drizzle/0009_loud_blue_marvel.sql b/apps/chain-indexer/drizzle/0009_loud_blue_marvel.sql new file mode 100644 index 0000000000..5b2d807efe --- /dev/null +++ b/apps/chain-indexer/drizzle/0009_loud_blue_marvel.sql @@ -0,0 +1,49 @@ +CREATE TABLE "akash"."daily_prices" ( + "date" date NOT NULL, + "denom" text NOT NULL, + "price" numeric(38, 18) NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "daily_prices_date_denom_pk" PRIMARY KEY("date","denom") +); +--> statement-breakpoint +CREATE TABLE "akash"."network_rollups" ( + "date" date PRIMARY KEY NOT NULL, + "close_height" bigint NOT NULL, + "close_at" timestamp with time zone NOT NULL, + "active_lease_count" integer NOT NULL, + "total_lease_count" bigint NOT NULL, + "daily_lease_count" integer NOT NULL, + "active_provider_count" integer NOT NULL, + "active_cpu_units" bigint NOT NULL, + "active_gpu_units" bigint NOT NULL, + "active_memory_bytes" bigint NOT NULL, + "active_ephemeral_storage_bytes" bigint NOT NULL, + "active_persistent_storage_bytes" bigint NOT NULL, + "total_uakt_spent" numeric(38, 18) NOT NULL, + "total_uusdc_spent" numeric(38, 18) NOT NULL, + "total_uact_spent" numeric(38, 18) NOT NULL, + "daily_uakt_spent" numeric(38, 18) NOT NULL, + "daily_uusdc_spent" numeric(38, 18) NOT NULL, + "daily_uact_spent" numeric(38, 18) NOT NULL, + "daily_usd_spent" numeric(38, 18), + "akt_price_used" numeric(38, 18), + "usd_computed_at" timestamp with time zone +); +--> statement-breakpoint +CREATE TABLE "akash"."network_state" ( + "id" integer PRIMARY KEY NOT NULL, + "last_aggregated_height" bigint NOT NULL, + "last_aggregated_at" timestamp with time zone NOT NULL, + "active_lease_count" integer DEFAULT 0 NOT NULL, + "total_lease_count" bigint DEFAULT 0 NOT NULL, + "active_provider_count" integer DEFAULT 0 NOT NULL, + "active_cpu_units" bigint DEFAULT 0 NOT NULL, + "active_gpu_units" bigint DEFAULT 0 NOT NULL, + "active_memory_bytes" bigint DEFAULT 0 NOT NULL, + "active_ephemeral_storage_bytes" bigint DEFAULT 0 NOT NULL, + "active_persistent_storage_bytes" bigint DEFAULT 0 NOT NULL, + "total_uakt_spent" numeric(38, 18) DEFAULT '0' NOT NULL, + "total_uusdc_spent" numeric(38, 18) DEFAULT '0' NOT NULL, + "total_uact_spent" numeric(38, 18) DEFAULT '0' NOT NULL, + CONSTRAINT "network_state_singleton_check" CHECK ("akash"."network_state"."id" = 1) +); diff --git a/apps/chain-indexer/drizzle/meta/0009_snapshot.json b/apps/chain-indexer/drizzle/meta/0009_snapshot.json new file mode 100644 index 0000000000..ef8433d738 --- /dev/null +++ b/apps/chain-indexer/drizzle/meta/0009_snapshot.json @@ -0,0 +1,3057 @@ +{ + "id": "080d0308-11dd-4220-855e-30746d3813e4", + "prevId": "2c31f1a9-1835-449a-899e-f6eaa1ebec5b", + "version": "7", + "dialect": "postgresql", + "tables": { + "cosmos.account_balances": { + "name": "account_balances", + "schema": "cosmos", + "columns": { + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "amount": { + "name": "amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "account_balances_account_id_accounts_id_fk": { + "name": "account_balances_account_id_accounts_id_fk", + "tableFrom": "account_balances", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_balances_account_id_denom_pk": { + "name": "account_balances_account_id_denom_pk", + "columns": [ + "account_id", + "denom" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.account_txs": { + "name": "account_txs", + "schema": "cosmos", + "columns": { + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "account_tx_role", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "account_txs_account_id_accounts_id_fk": { + "name": "account_txs_account_id_accounts_id_fk", + "tableFrom": "account_txs", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_txs_account_id_height_tx_index_role_pk": { + "name": "account_txs_account_id_height_tx_index_role_pk", + "columns": [ + "account_id", + "height", + "tx_index", + "role" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.accounts": { + "name": "accounts", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_number": { + "name": "account_number", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "account_type": { + "name": "account_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_module_account": { + "name": "is_module_account", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": { + "accounts_address_idx": { + "name": "accounts_address_idx", + "columns": [ + { + "expression": "address", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.balance_changes": { + "name": "balance_changes", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delta": { + "name": "delta", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "balance_after": { + "name": "balance_after", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "balance_change_reason", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "event_index": { + "name": "event_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "counterparty_account_id": { + "name": "counterparty_account_id", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "balance_changes_account_denom_height_idx": { + "name": "balance_changes_account_denom_height_idx", + "columns": [ + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "denom", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "balance_changes_height_event_index_idx": { + "name": "balance_changes_height_event_index_idx", + "columns": [ + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_index", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "balance_changes_account_id_accounts_id_fk": { + "name": "balance_changes_account_id_accounts_id_fk", + "tableFrom": "balance_changes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "balance_changes_counterparty_account_id_accounts_id_fk": { + "name": "balance_changes_counterparty_account_id_accounts_id_fk", + "tableFrom": "balance_changes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "counterparty_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.bids": { + "name": "bids", + "schema": "akash", + "columns": { + "deployment_id": { + "name": "deployment_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gseq": { + "name": "gseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "oseq": { + "name": "oseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "bseq": { + "name": "bseq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "provider_account_id": { + "name": "provider_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "price": { + "name": "price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "bid_state", + "typeSchema": "akash", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "created_height": { + "name": "created_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "closed_height": { + "name": "closed_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "bids_deployment_id_deployments_id_fk": { + "name": "bids_deployment_id_deployments_id_fk", + "tableFrom": "bids", + "tableTo": "deployments", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "bids_provider_account_id_accounts_id_fk": { + "name": "bids_provider_account_id_accounts_id_fk", + "tableFrom": "bids", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "provider_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "bids_deployment_id_gseq_oseq_bseq_provider_account_id_pk": { + "name": "bids_deployment_id_gseq_oseq_bseq_provider_account_id_pk", + "columns": [ + "deployment_id", + "gseq", + "oseq", + "bseq", + "provider_account_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.blocks": { + "name": "blocks", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "datetime": { + "name": "datetime", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "parent_hash": { + "name": "parent_hash", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "proposer_address": { + "name": "proposer_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tx_count": { + "name": "tx_count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.bme_canceled_records": { + "name": "bme_canceled_records", + "schema": "akash", + "columns": { + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "to_denom": { + "name": "to_denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "record_height": { + "name": "record_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "sequence": { + "name": "sequence", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cancel_reason": { + "name": "cancel_reason", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "owner_account_id": { + "name": "owner_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "to_account_id": { + "name": "to_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "coins_to_burn_denom": { + "name": "coins_to_burn_denom", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "coins_to_burn_amount": { + "name": "coins_to_burn_amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "denom_to_mint": { + "name": "denom_to_mint", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "bme_canceled_records_height_idx": { + "name": "bme_canceled_records_height_idx", + "columns": [ + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "bme_canceled_records_owner_account_id_accounts_id_fk": { + "name": "bme_canceled_records_owner_account_id_accounts_id_fk", + "tableFrom": "bme_canceled_records", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "owner_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "bme_canceled_records_to_account_id_accounts_id_fk": { + "name": "bme_canceled_records_to_account_id_accounts_id_fk", + "tableFrom": "bme_canceled_records", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "to_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "bme_canceled_records_record_height_sequence_denom_to_denom_source_pk": { + "name": "bme_canceled_records_record_height_sequence_denom_to_denom_source_pk", + "columns": [ + "record_height", + "sequence", + "denom", + "to_denom", + "source" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.bme_ledger_records": { + "name": "bme_ledger_records", + "schema": "akash", + "columns": { + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "to_denom": { + "name": "to_denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "record_height": { + "name": "record_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "sequence": { + "name": "sequence", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "burned_from_account_id": { + "name": "burned_from_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "minted_to_account_id": { + "name": "minted_to_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "burned_denom": { + "name": "burned_denom", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "burned_amount": { + "name": "burned_amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "burned_price": { + "name": "burned_price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": false + }, + "minted_denom": { + "name": "minted_denom", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "minted_amount": { + "name": "minted_amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "minted_price": { + "name": "minted_price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": false + }, + "spread_denom": { + "name": "spread_denom", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "spread_amount": { + "name": "spread_amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "remint_credit_issued_amount": { + "name": "remint_credit_issued_amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "remint_credit_accrued_amount": { + "name": "remint_credit_accrued_amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "bme_ledger_records_height_idx": { + "name": "bme_ledger_records_height_idx", + "columns": [ + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "bme_ledger_records_burned_denom_height_idx": { + "name": "bme_ledger_records_burned_denom_height_idx", + "columns": [ + { + "expression": "burned_denom", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "bme_ledger_records_minted_denom_height_idx": { + "name": "bme_ledger_records_minted_denom_height_idx", + "columns": [ + { + "expression": "minted_denom", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "bme_ledger_records_burned_from_account_id_accounts_id_fk": { + "name": "bme_ledger_records_burned_from_account_id_accounts_id_fk", + "tableFrom": "bme_ledger_records", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "burned_from_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "bme_ledger_records_minted_to_account_id_accounts_id_fk": { + "name": "bme_ledger_records_minted_to_account_id_accounts_id_fk", + "tableFrom": "bme_ledger_records", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "minted_to_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "bme_ledger_records_record_height_sequence_denom_to_denom_source_pk": { + "name": "bme_ledger_records_record_height_sequence_denom_to_denom_source_pk", + "columns": [ + "record_height", + "sequence", + "denom", + "to_denom", + "source" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.bme_status_changes": { + "name": "bme_status_changes", + "schema": "akash", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ordinal": { + "name": "ordinal", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "previous_status": { + "name": "previous_status", + "type": "bme_mint_status", + "typeSchema": "akash", + "primaryKey": false, + "notNull": true + }, + "new_status": { + "name": "new_status", + "type": "bme_mint_status", + "typeSchema": "akash", + "primaryKey": false, + "notNull": true + }, + "collateral_ratio": { + "name": "collateral_ratio", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "bme_status_changes_height_ordinal_pk": { + "name": "bme_status_changes_height_ordinal_pk", + "columns": [ + "height", + "ordinal" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.daily_prices": { + "name": "daily_prices", + "schema": "akash", + "columns": { + "date": { + "name": "date", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "price": { + "name": "price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "daily_prices_date_denom_pk": { + "name": "daily_prices_date_denom_pk", + "columns": [ + "date", + "denom" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.delegations": { + "name": "delegations", + "schema": "cosmos", + "columns": { + "delegator_account_id": { + "name": "delegator_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "validator_operator_address": { + "name": "validator_operator_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "shares": { + "name": "shares", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "delegations_delegator_account_id_accounts_id_fk": { + "name": "delegations_delegator_account_id_accounts_id_fk", + "tableFrom": "delegations", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "delegator_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "delegations_delegator_account_id_validator_operator_address_pk": { + "name": "delegations_delegator_account_id_validator_operator_address_pk", + "columns": [ + "delegator_account_id", + "validator_operator_address" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.deployment_events": { + "name": "deployment_events", + "schema": "akash", + "columns": { + "deployment_id": { + "name": "deployment_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ordinal": { + "name": "ordinal", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "msg_index": { + "name": "msg_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "deployment_event_type", + "typeSchema": "akash", + "primaryKey": false, + "notNull": true + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "deployment_events_deployment_id_deployments_id_fk": { + "name": "deployment_events_deployment_id_deployments_id_fk", + "tableFrom": "deployment_events", + "tableTo": "deployments", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "deployment_events_deployment_id_height_ordinal_pk": { + "name": "deployment_events_deployment_id_height_ordinal_pk", + "columns": [ + "deployment_id", + "height", + "ordinal" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.deployment_group_resources": { + "name": "deployment_group_resources", + "schema": "akash", + "columns": { + "deployment_group_id": { + "name": "deployment_group_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "idx": { + "name": "idx", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "cpu_units": { + "name": "cpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gpu_units": { + "name": "gpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gpu_vendor": { + "name": "gpu_vendor", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "gpu_model": { + "name": "gpu_model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "memory_bytes": { + "name": "memory_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ephemeral_storage_bytes": { + "name": "ephemeral_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "persistent_storage_bytes": { + "name": "persistent_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "price": { + "name": "price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "price_denom": { + "name": "price_denom", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "deployment_group_resources_deployment_group_id_deployment_groups_id_fk": { + "name": "deployment_group_resources_deployment_group_id_deployment_groups_id_fk", + "tableFrom": "deployment_group_resources", + "tableTo": "deployment_groups", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_group_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "deployment_group_resources_deployment_group_id_idx_pk": { + "name": "deployment_group_resources_deployment_group_id_idx_pk", + "columns": [ + "deployment_group_id", + "idx" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.deployment_groups": { + "name": "deployment_groups", + "schema": "akash", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "deployment_id": { + "name": "deployment_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gseq": { + "name": "gseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "group_state", + "typeSchema": "akash", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "closed_height": { + "name": "closed_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "deployment_groups_deployment_gseq_idx": { + "name": "deployment_groups_deployment_gseq_idx", + "columns": [ + { + "expression": "deployment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "gseq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "deployment_groups_deployment_id_deployments_id_fk": { + "name": "deployment_groups_deployment_id_deployments_id_fk", + "tableFrom": "deployment_groups", + "tableTo": "deployments", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.deployments": { + "name": "deployments", + "schema": "akash", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "owner_account_id": { + "name": "owner_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "dseq": { + "name": "dseq", + "type": "numeric(20, 0)", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deposit": { + "name": "deposit", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "balance": { + "name": "balance", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "withdrawn_amount": { + "name": "withdrawn_amount", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "block_rate": { + "name": "block_rate", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "last_withdraw_height": { + "name": "last_withdraw_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "last_processed_height": { + "name": "last_processed_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_height": { + "name": "created_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "closed_height": { + "name": "closed_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "closed_at": { + "name": "closed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "close_reason": { + "name": "close_reason", + "type": "deployment_close_reason", + "typeSchema": "akash", + "primaryKey": false, + "notNull": false + }, + "cpu_units": { + "name": "cpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gpu_units": { + "name": "gpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "memory_bytes": { + "name": "memory_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ephemeral_storage_bytes": { + "name": "ephemeral_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "persistent_storage_bytes": { + "name": "persistent_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "deployments_owner_dseq_idx": { + "name": "deployments_owner_dseq_idx", + "columns": [ + { + "expression": "owner_account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dseq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "deployments_owner_created_idx": { + "name": "deployments_owner_created_idx", + "columns": [ + { + "expression": "owner_account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "deployments_open_idx": { + "name": "deployments_open_idx", + "columns": [ + { + "expression": "created_height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"akash\".\"deployments\".\"closed_height\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "deployments_owner_account_id_accounts_id_fk": { + "name": "deployments_owner_account_id_accounts_id_fk", + "tableFrom": "deployments", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "owner_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.indexer_state": { + "name": "indexer_state", + "schema": "", + "columns": { + "stream": { + "name": "stream", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "last_height": { + "name": "last_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.leases": { + "name": "leases", + "schema": "akash", + "columns": { + "deployment_id": { + "name": "deployment_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "deployment_group_id": { + "name": "deployment_group_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gseq": { + "name": "gseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "oseq": { + "name": "oseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "bseq": { + "name": "bseq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "provider_account_id": { + "name": "provider_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "price": { + "name": "price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "balance": { + "name": "balance", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "withdrawn_amount": { + "name": "withdrawn_amount", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "predicted_closed_height": { + "name": "predicted_closed_height", + "type": "numeric(30, 0)", + "primaryKey": false, + "notNull": true + }, + "created_height": { + "name": "created_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "closed_height": { + "name": "closed_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "closed_at": { + "name": "closed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "cpu_units": { + "name": "cpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gpu_units": { + "name": "gpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "memory_bytes": { + "name": "memory_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ephemeral_storage_bytes": { + "name": "ephemeral_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "persistent_storage_bytes": { + "name": "persistent_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "leases_provider_idx": { + "name": "leases_provider_idx", + "columns": [ + { + "expression": "provider_account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "closed_height", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "leases_open_idx": { + "name": "leases_open_idx", + "columns": [ + { + "expression": "deployment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"akash\".\"leases\".\"closed_height\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "leases_deployment_id_deployments_id_fk": { + "name": "leases_deployment_id_deployments_id_fk", + "tableFrom": "leases", + "tableTo": "deployments", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "leases_deployment_group_id_deployment_groups_id_fk": { + "name": "leases_deployment_group_id_deployment_groups_id_fk", + "tableFrom": "leases", + "tableTo": "deployment_groups", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_group_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "leases_provider_account_id_accounts_id_fk": { + "name": "leases_provider_account_id_accounts_id_fk", + "tableFrom": "leases", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "provider_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "leases_deployment_id_gseq_oseq_bseq_provider_account_id_pk": { + "name": "leases_deployment_id_gseq_oseq_bseq_provider_account_id_pk", + "columns": [ + "deployment_id", + "gseq", + "oseq", + "bseq", + "provider_account_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.message_dead_letters": { + "name": "message_dead_letters", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type_id": { + "name": "type_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "raw": { + "name": "raw", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "message_dead_letters_type_id_idx": { + "name": "message_dead_letters_type_id_idx", + "columns": [ + { + "expression": "type_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "message_dead_letters_type_id_message_types_id_fk": { + "name": "message_dead_letters_type_id_message_types_id_fk", + "tableFrom": "message_dead_letters", + "tableTo": "message_types", + "schemaTo": "cosmos", + "columnsFrom": [ + "type_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "message_dead_letters_height_tx_index_index_pk": { + "name": "message_dead_letters_height_tx_index_index_pk", + "columns": [ + "height", + "tx_index", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.message_types": { + "name": "message_types", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "message_types_type_idx": { + "name": "message_types_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.messages": { + "name": "messages", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type_id": { + "name": "type_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "body": { + "name": "body", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "messages_type_id_idx": { + "name": "messages_type_id_idx", + "columns": [ + { + "expression": "type_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "messages_type_id_message_types_id_fk": { + "name": "messages_type_id_message_types_id_fk", + "tableFrom": "messages", + "tableTo": "message_types", + "schemaTo": "cosmos", + "columnsFrom": [ + "type_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messages_height_tx_index_index_pk": { + "name": "messages_height_tx_index_index_pk", + "columns": [ + "height", + "tx_index", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.network_rollups": { + "name": "network_rollups", + "schema": "akash", + "columns": { + "date": { + "name": "date", + "type": "date", + "primaryKey": true, + "notNull": true + }, + "close_height": { + "name": "close_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "close_at": { + "name": "close_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "active_lease_count": { + "name": "active_lease_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "total_lease_count": { + "name": "total_lease_count", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "daily_lease_count": { + "name": "daily_lease_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "active_provider_count": { + "name": "active_provider_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "active_cpu_units": { + "name": "active_cpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "active_gpu_units": { + "name": "active_gpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "active_memory_bytes": { + "name": "active_memory_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "active_ephemeral_storage_bytes": { + "name": "active_ephemeral_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "active_persistent_storage_bytes": { + "name": "active_persistent_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "total_uakt_spent": { + "name": "total_uakt_spent", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "total_uusdc_spent": { + "name": "total_uusdc_spent", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "total_uact_spent": { + "name": "total_uact_spent", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "daily_uakt_spent": { + "name": "daily_uakt_spent", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "daily_uusdc_spent": { + "name": "daily_uusdc_spent", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "daily_uact_spent": { + "name": "daily_uact_spent", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "daily_usd_spent": { + "name": "daily_usd_spent", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": false + }, + "akt_price_used": { + "name": "akt_price_used", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": false + }, + "usd_computed_at": { + "name": "usd_computed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.network_state": { + "name": "network_state", + "schema": "akash", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "last_aggregated_height": { + "name": "last_aggregated_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "last_aggregated_at": { + "name": "last_aggregated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "active_lease_count": { + "name": "active_lease_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_lease_count": { + "name": "total_lease_count", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "active_provider_count": { + "name": "active_provider_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "active_cpu_units": { + "name": "active_cpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "active_gpu_units": { + "name": "active_gpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "active_memory_bytes": { + "name": "active_memory_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "active_ephemeral_storage_bytes": { + "name": "active_ephemeral_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "active_persistent_storage_bytes": { + "name": "active_persistent_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_uakt_spent": { + "name": "total_uakt_spent", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "total_uusdc_spent": { + "name": "total_uusdc_spent", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "total_uact_spent": { + "name": "total_uact_spent", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "network_state_singleton_check": { + "name": "network_state_singleton_check", + "value": "\"akash\".\"network_state\".\"id\" = 1" + } + }, + "isRLSEnabled": false + }, + "cosmos.proposal_deposits": { + "name": "proposal_deposits", + "schema": "cosmos", + "columns": { + "proposal_id": { + "name": "proposal_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "depositor_account_id": { + "name": "depositor_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "amount": { + "name": "amount", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "proposal_deposits_depositor_account_id_accounts_id_fk": { + "name": "proposal_deposits_depositor_account_id_accounts_id_fk", + "tableFrom": "proposal_deposits", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "depositor_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "proposal_deposits_proposal_id_depositor_account_id_height_pk": { + "name": "proposal_deposits_proposal_id_depositor_account_id_height_pk", + "columns": [ + "proposal_id", + "depositor_account_id", + "height" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.proposal_votes": { + "name": "proposal_votes", + "schema": "cosmos", + "columns": { + "proposal_id": { + "name": "proposal_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "voter_account_id": { + "name": "voter_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "options": { + "name": "options", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "proposal_votes_voter_account_id_accounts_id_fk": { + "name": "proposal_votes_voter_account_id_accounts_id_fk", + "tableFrom": "proposal_votes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "voter_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "proposal_votes_proposal_id_voter_account_id_pk": { + "name": "proposal_votes_proposal_id_voter_account_id_pk", + "columns": [ + "proposal_id", + "voter_account_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.proposals": { + "name": "proposals", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "proposer_account_id": { + "name": "proposer_account_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "messages": { + "name": "messages", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "proposal_status", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + }, + "submit_time": { + "name": "submit_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deposit_end_time": { + "name": "deposit_end_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "voting_start_time": { + "name": "voting_start_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "voting_end_time": { + "name": "voting_end_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "total_deposit": { + "name": "total_deposit", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "final_tally_yes": { + "name": "final_tally_yes", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "final_tally_abstain": { + "name": "final_tally_abstain", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "final_tally_no": { + "name": "final_tally_no", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "final_tally_no_with_veto": { + "name": "final_tally_no_with_veto", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "submit_height": { + "name": "submit_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "proposals_proposer_account_id_accounts_id_fk": { + "name": "proposals_proposer_account_id_accounts_id_fk", + "tableFrom": "proposals", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "proposer_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.provider_audit_signatures": { + "name": "provider_audit_signatures", + "schema": "akash", + "columns": { + "owner_account_id": { + "name": "owner_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "auditor_account_id": { + "name": "auditor_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "provider_audit_signatures_owner_account_id_accounts_id_fk": { + "name": "provider_audit_signatures_owner_account_id_accounts_id_fk", + "tableFrom": "provider_audit_signatures", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "owner_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "provider_audit_signatures_auditor_account_id_accounts_id_fk": { + "name": "provider_audit_signatures_auditor_account_id_accounts_id_fk", + "tableFrom": "provider_audit_signatures", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "auditor_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "provider_audit_signatures_owner_account_id_auditor_account_id_key_pk": { + "name": "provider_audit_signatures_owner_account_id_auditor_account_id_key_pk", + "columns": [ + "owner_account_id", + "auditor_account_id", + "key" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.providers": { + "name": "providers", + "schema": "akash", + "columns": { + "owner_account_id": { + "name": "owner_account_id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "host_uri": { + "name": "host_uri", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attributes": { + "name": "attributes", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "last_processed_height": { + "name": "last_processed_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_height": { + "name": "created_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "updated_height": { + "name": "updated_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "deleted_height": { + "name": "deleted_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "providers_owner_account_id_accounts_id_fk": { + "name": "providers_owner_account_id_accounts_id_fk", + "tableFrom": "providers", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "owner_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.transactions": { + "name": "transactions", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "code": { + "name": "code", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "gas_used": { + "name": "gas_used", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gas_wanted": { + "name": "gas_wanted", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "fee": { + "name": "fee", + "type": "jsonb", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "transactions_hash_idx": { + "name": "transactions_hash_idx", + "columns": [ + { + "expression": "hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "transactions_height_index_pk": { + "name": "transactions_height_index_pk", + "columns": [ + "height", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.unbonding_delegations": { + "name": "unbonding_delegations", + "schema": "cosmos", + "columns": { + "delegator_account_id": { + "name": "delegator_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "validator_operator_address": { + "name": "validator_operator_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "creation_height": { + "name": "creation_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "completion_time": { + "name": "completion_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "initial_balance": { + "name": "initial_balance", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "balance": { + "name": "balance", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "unbonding_delegations_delegator_account_id_accounts_id_fk": { + "name": "unbonding_delegations_delegator_account_id_accounts_id_fk", + "tableFrom": "unbonding_delegations", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "delegator_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "unbonding_delegations_delegator_account_id_validator_operator_address_creation_height_pk": { + "name": "unbonding_delegations_delegator_account_id_validator_operator_address_creation_height_pk", + "columns": [ + "delegator_account_id", + "validator_operator_address", + "creation_height" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.validators": { + "name": "validators", + "schema": "cosmos", + "columns": { + "operator_address": { + "name": "operator_address", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_address": { + "name": "account_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "hex_address": { + "name": "hex_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "moniker": { + "name": "moniker", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "identity": { + "name": "identity", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "details": { + "name": "details", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "security_contact": { + "name": "security_contact", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commission_rate": { + "name": "commission_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "commission_max_rate": { + "name": "commission_max_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "commission_max_change_rate": { + "name": "commission_max_change_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "min_self_delegation": { + "name": "min_self_delegation", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "jailed": { + "name": "jailed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "status": { + "name": "status", + "type": "validator_status", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": false + }, + "tokens": { + "name": "tokens", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "delegator_shares": { + "name": "delegator_shares", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": false + }, + "unbonding_height": { + "name": "unbonding_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "unbonding_time": { + "name": "unbonding_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "cosmos.account_tx_role": { + "name": "account_tx_role", + "schema": "cosmos", + "values": [ + "signer", + "sender", + "receiver" + ] + }, + "cosmos.balance_change_reason": { + "name": "balance_change_reason", + "schema": "cosmos", + "values": [ + "genesis", + "transfer", + "fee", + "reward", + "commission", + "slash", + "gov", + "ibc", + "escrow", + "bme", + "mint", + "burn", + "staking" + ] + }, + "akash.bid_state": { + "name": "bid_state", + "schema": "akash", + "values": [ + "open", + "active", + "closed" + ] + }, + "akash.bme_mint_status": { + "name": "bme_mint_status", + "schema": "akash", + "values": [ + "mint_status_unspecified", + "mint_status_healthy", + "mint_status_warning", + "mint_status_halt_cr", + "mint_status_halt_oracle" + ] + }, + "akash.deployment_close_reason": { + "name": "deployment_close_reason", + "schema": "akash", + "values": [ + "close_message", + "overdrawn", + "close_event" + ] + }, + "akash.deployment_event_type": { + "name": "deployment_event_type", + "schema": "akash", + "values": [ + "created", + "deposited", + "updated", + "closed", + "group_closed", + "group_paused", + "group_started", + "bid_created", + "bid_closed", + "lease_created", + "lease_closed", + "lease_withdrawn" + ] + }, + "akash.group_state": { + "name": "group_state", + "schema": "akash", + "values": [ + "open", + "paused", + "closed" + ] + }, + "cosmos.proposal_status": { + "name": "proposal_status", + "schema": "cosmos", + "values": [ + "deposit_period", + "voting_period", + "passed", + "rejected", + "failed" + ] + }, + "cosmos.validator_status": { + "name": "validator_status", + "schema": "cosmos", + "values": [ + "unbonded", + "unbonding", + "bonded" + ] + }, + "cosmos.vote_option": { + "name": "vote_option", + "schema": "cosmos", + "values": [ + "yes", + "abstain", + "no", + "no_with_veto" + ] + } + }, + "schemas": { + "akash": "akash", + "cosmos": "cosmos" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/chain-indexer/drizzle/meta/_journal.json b/apps/chain-indexer/drizzle/meta/_journal.json index 19bebd6d6b..669c3151c7 100644 --- a/apps/chain-indexer/drizzle/meta/_journal.json +++ b/apps/chain-indexer/drizzle/meta/_journal.json @@ -64,6 +64,13 @@ "when": 1786949585273, "tag": "0008_good_galactus", "breakpoints": true + }, + { + "idx": 9, + "version": "7", + "when": 1786993129072, + "tag": "0009_loud_blue_marvel", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/chain-indexer/package.json b/apps/chain-indexer/package.json index dfb1a7fc76..3f4921d4b6 100644 --- a/apps/chain-indexer/package.json +++ b/apps/chain-indexer/package.json @@ -12,6 +12,7 @@ "format": "prettier --write ./*.{ts,js,json} **/*.{ts,js,json}", "lint": "eslint .", "migration:gen": "drizzle-kit generate", + "network:recompute-usd": "npm run build && node --enable-source-maps ./dist/recompute-usd.js", "prod": "node --enable-source-maps --require ./dist/instrumentation.js ./dist/server.js", "reconcile": "npm run build && node --enable-source-maps ./dist/reconcile.js", "start": "tsup --watch", diff --git a/apps/chain-indexer/src/akash/akash-writer.service.spec.ts b/apps/chain-indexer/src/akash/akash-writer.service.spec.ts index fa1e0fb5d9..79de36991e 100644 --- a/apps/chain-indexer/src/akash/akash-writer.service.spec.ts +++ b/apps/chain-indexer/src/akash/akash-writer.service.spec.ts @@ -5,6 +5,7 @@ import { mock } from "vitest-mock-extended"; import type { AkashBlockChanges, AkashChangeBody } from "@src/akash/akash-changes"; import { AkashWriter } from "@src/akash/akash-writer.service"; +import { decFromInt } from "@src/akash/dec"; import { Bids, DeploymentEvents, DeploymentGroupResources, DeploymentGroups, Deployments, Leases } from "@src/db/schema"; import type { ChainTransaction } from "@src/providers/db.provider"; import type { LoggerService } from "@src/providers/logging.provider"; @@ -23,16 +24,17 @@ describe(AkashWriter.name, () => { it("does nothing for blocks without akash changes", async () => { const { writer, tx, inserts, selects } = setup(); - await writer.write(tx, [block(100, [])], ACCOUNT_IDS); + const { networkDeltas } = await writer.write(tx, [block(100, [])], ACCOUNT_IDS); expect(inserts).toEqual([]); expect(selects).toEqual([]); + expect(networkDeltas).toEqual([]); }); it("persists a full lifecycle batch as one consistent set of rows", async () => { const { writer, tx, inserts, upserts } = setup(); - await writer.write( + const { networkDeltas } = await writer.write( tx, [block(100, [create(), bidCreated("10")]), block(110, [{ kind: "leaseCreated", key: LEASE_KEY }]), block(200, [{ kind: "deploymentClosed", key: KEY }])], ACCOUNT_IDS @@ -83,6 +85,11 @@ describe(AkashWriter.name, () => { const deploymentUpsert = upserts.find(upsert => upsert.table === Deployments); expect(whereSql(deploymentUpsert?.config.setWhere as SQL)).toContain('excluded.last_processed_height >= "akash"."deployments"."last_processed_height"'); + + expect(networkDeltas).toEqual([ + expect.objectContaining({ height: 110, leasesCreated: 1, activeLeaseDelta: 1, cpuUnitsDelta: 2000, earnedDeltaByDenom: new Map() }), + expect.objectContaining({ height: 200, activeLeaseDelta: -1, cpuUnitsDelta: -2000, earnedDeltaByDenom: new Map([["uakt", decFromInt(900)]]) }) + ]); }); it("skips flushing entirely when every block is at or below the stored watermark", async () => { @@ -90,15 +97,16 @@ describe(AkashWriter.name, () => { deployments: [deploymentRow({ lastProcessedHeight: 500 })] }); - await writer.write(tx, [block(400, [{ kind: "deploymentDeposited", key: KEY, amount: "10", depositor: null }])], ACCOUNT_IDS); + const { networkDeltas } = await writer.write(tx, [block(400, [{ kind: "deploymentDeposited", key: KEY, amount: "10", depositor: null }])], ACCOUNT_IDS); expect(inserts).toEqual([]); + expect(networkDeltas).toEqual([]); }); it("ignores provider changes entirely", async () => { const { writer, tx, inserts, selects, logger } = setup(); - await writer.write( + const { networkDeltas } = await writer.write( tx, [block(100, [{ kind: "providerCreated", owner: OWNER, hostUri: "https://x", email: null, website: null, attributes: [] }])], ACCOUNT_IDS @@ -106,6 +114,7 @@ describe(AkashWriter.name, () => { expect(inserts).toEqual([]); expect(selects).toEqual([]); + expect(networkDeltas).toEqual([]); expect(logger.warn).not.toHaveBeenCalled(); }); diff --git a/apps/chain-indexer/src/akash/akash-writer.service.ts b/apps/chain-indexer/src/akash/akash-writer.service.ts index 9ff7faf6d7..c28ae52036 100644 --- a/apps/chain-indexer/src/akash/akash-writer.service.ts +++ b/apps/chain-indexer/src/akash/akash-writer.service.ts @@ -7,6 +7,8 @@ import { isProviderChange } from "@src/akash/akash-changes"; import { decFromString, decToString } from "@src/akash/dec"; import type { BidStateValue, DeploymentAggState, GroupStateValue, ReducerWarning } from "@src/akash/deployment-reducer"; import { applyBlockChanges, stateKey } from "@src/akash/deployment-reducer"; +import type { NetworkBlockDelta } from "@src/akash/network-delta"; +import { diffNetworkDelta, isEmptyNetworkDelta, snapshotNetworkState } from "@src/akash/network-delta"; import { sumLeaseRate } from "@src/akash/settlement"; import { insertChunked } from "@src/db/insert-chunked"; import { Accounts, Bids, DeploymentEvents, DeploymentGroupResources, DeploymentGroups, Deployments, Leases } from "@src/db/schema"; @@ -37,24 +39,33 @@ export class AkashWriter { this.#logger.setContext("AKASH_WRITER"); } - async write(tx: ChainTransaction, blocks: AkashBlockChanges[], accountIds: Map): Promise { + async write(tx: ChainTransaction, blocks: AkashBlockChanges[], accountIds: Map): Promise<{ networkDeltas: NetworkBlockDelta[] }> { const withChanges = blocks.filter(block => block.changes.length > 0); if (withChanges.length === 0) { - return; + return { networkDeltas: [] }; } const keyed = this.#collectKeys(withChanges, accountIds); if (keyed.length === 0) { - return; + return { networkDeltas: [] }; } const { states, deploymentIds, groupIds, loadedAddressIds } = await this.#loadStates(tx, keyed); - const warnings = withChanges.flatMap(block => applyBlockChanges(states, block)); + const warnings: ReducerWarning[] = []; + const networkDeltas: NetworkBlockDelta[] = []; + for (const block of withChanges) { + const before = snapshotNetworkState(states, block); + warnings.push(...applyBlockChanges(states, block)); + const delta = diffNetworkDelta(before, states, block); + if (!isEmptyNetworkDelta(delta)) { + networkDeltas.push(delta); + } + } this.#logWarnings(warnings); const touched = [...states.values()].filter(state => state.touched); if (touched.length === 0) { - return; + return { networkDeltas }; } /** Providers of bids and leases loaded from prior batches aren't in this batch's interned map, so their ids come from the loaded rows. */ @@ -66,6 +77,8 @@ export class AkashWriter { await this.#flushBids(tx, touched, addressIds, deploymentIds); await this.#flushLeases(tx, touched, addressIds, deploymentIds, groupIds); await this.#flushEvents(tx, touched, deploymentIds); + + return { networkDeltas }; } /** Deterministic (ownerAccountId, dseq) order for both the row locks and the flush statements, so concurrent writers cannot deadlock. */ diff --git a/apps/chain-indexer/src/akash/network-delta.spec.ts b/apps/chain-indexer/src/akash/network-delta.spec.ts new file mode 100644 index 0000000000..13ace54de4 --- /dev/null +++ b/apps/chain-indexer/src/akash/network-delta.spec.ts @@ -0,0 +1,198 @@ +import { describe, expect, it } from "vitest"; + +import type { AkashBlockChanges, AkashChangeBody, NormalizedGroup } from "@src/akash/akash-changes"; +import { decFromInt } from "@src/akash/dec"; +import type { DeploymentAggState } from "@src/akash/deployment-reducer"; +import { applyBlockChanges } from "@src/akash/deployment-reducer"; +import type { NetworkBlockDelta } from "@src/akash/network-delta"; +import { diffNetworkDelta, isEmptyNetworkDelta, snapshotNetworkState } from "@src/akash/network-delta"; + +const OWNER = "akash1owner"; +const PROVIDER = "akash1prov"; +const KEY = { owner: OWNER, dseq: "42" }; +const LEASE_KEY = { ...KEY, gseq: 1, oseq: 1, bseq: 0, provider: PROVIDER }; +const BLOCK_TIME = new Date("2026-08-13T00:00:00Z"); + +describe("networkDelta", () => { + it("counts a created lease with its own group's resource totals", () => { + const { states, fold } = setup(); + fold(block(100, [create({ groups: twoGroups() }), bidCreated("10")])); + + const delta = fold(block(110, [leaseCreated()])); + + expect(delta).toEqual({ + height: 110, + leasesCreated: 1, + activeLeaseDelta: 1, + cpuUnitsDelta: 1000 * 2, + gpuUnitsDelta: 0 * 2, + memoryBytesDelta: 1024 * 2, + ephemeralStorageBytesDelta: 100 * 2, + persistentStorageBytesDelta: 50 * 2, + earnedDeltaByDenom: new Map() + }); + expect(states.size).toBe(1); + }); + + it("removes resources and recognizes settled earnings when a lease closes", () => { + const { fold } = setup(); + fold(block(100, [create({}), bidCreated("10")])); + fold(block(110, [leaseCreated()])); + + const delta = fold(block(120, [{ kind: "leaseClosed", key: LEASE_KEY }])); + + expect(delta.leasesCreated).toBe(0); + expect(delta.activeLeaseDelta).toBe(-1); + expect(delta.cpuUnitsDelta).toBe(-2000); + expect(delta.earnedDeltaByDenom).toEqual(new Map([["uakt", decFromInt(10 * 10)]])); + }); + + it("nets out resources but still counts the creation when a lease opens and closes in one block", () => { + const { fold } = setup(); + fold(block(100, [create({}), bidCreated("10")])); + + const delta = fold(block(110, [leaseCreated(), { kind: "leaseClosed", key: LEASE_KEY }])); + + expect(delta.leasesCreated).toBe(1); + expect(delta.activeLeaseDelta).toBe(0); + expect(delta.cpuUnitsDelta).toBe(0); + expect(delta.gpuUnitsDelta).toBe(0); + }); + + it("recognizes accrued earnings on withdrawal without touching resources", () => { + const { fold } = setup(); + fold(block(100, [create({}), bidCreated("10")])); + fold(block(110, [leaseCreated()])); + + const delta = fold(block(150, [{ kind: "leaseWithdrawn", key: LEASE_KEY }])); + + expect(delta.activeLeaseDelta).toBe(0); + expect(delta.cpuUnitsDelta).toBe(0); + expect(delta.earnedDeltaByDenom).toEqual(new Map([["uakt", decFromInt(40 * 10)]])); + }); + + it("excludes the close-time fractional refund from earnings", () => { + const { fold } = setup(); + fold(block(100, [create({ deposit: "500000" }), bidCreated("2.349334")])); + fold(block(110, [leaseCreated()])); + + const delta = fold(block(122, [{ kind: "leaseClosed", key: LEASE_KEY }])); + + expect(delta.earnedDeltaByDenom).toEqual(new Map([["uakt", decFromInt(28)]])); + }); + + it("keeps earnings of concurrent deployments separate per denom", () => { + const { fold } = setup(); + const usdcKey = { owner: OWNER, dseq: "43" }; + const usdcLeaseKey = { ...usdcKey, gseq: 1, oseq: 1, bseq: 0, provider: PROVIDER }; + fold( + block(100, [ + create({}), + bidCreated("10"), + { ...create({ denom: "ibc/170C677610AC31DF0904FFE09CD3B5C657492170E7E52372E48756B71E56F2F1" }), key: usdcKey }, + { kind: "bidCreated", key: usdcLeaseKey, price: "5", priceDenom: "uusdc" } + ]) + ); + fold(block(110, [leaseCreated(), { kind: "leaseCreated", key: usdcLeaseKey }])); + + const delta = fold( + block(120, [ + { kind: "leaseWithdrawn", key: LEASE_KEY }, + { kind: "leaseWithdrawn", key: usdcLeaseKey } + ]) + ); + + expect(delta.earnedDeltaByDenom).toEqual( + new Map([ + ["uakt", decFromInt(100)], + ["uusdc", decFromInt(50)] + ]) + ); + }); + + it("produces an empty delta for a provider-only block", () => { + const { states } = setup(); + + const before = snapshotNetworkState(states, block(100, [{ kind: "providerDeleted", owner: OWNER }])); + applyBlockChanges(states, block(100, [{ kind: "providerDeleted", owner: OWNER }])); + const delta = diffNetworkDelta(before, states, block(100, [])); + + expect(before.size).toBe(0); + expect(isEmptyNetworkDelta(delta)).toBe(true); + }); + + it("produces an empty delta when the watermark skips a replayed block", () => { + const { fold } = setup(); + fold(block(100, [create({}), bidCreated("10")])); + fold(block(110, [leaseCreated()])); + + const delta = fold(block(110, [leaseCreated()])); + + expect(isEmptyNetworkDelta(delta)).toBe(true); + }); + + function setup() { + const states = new Map(); + const fold = (blockChanges: AkashBlockChanges): NetworkBlockDelta => { + const before = snapshotNetworkState(states, blockChanges); + applyBlockChanges(states, blockChanges); + return diffNetworkDelta(before, states, blockChanges); + }; + return { states, fold }; + } + + function block(height: number, bodies: AkashChangeBody[]): AkashBlockChanges { + return { + height, + datetime: BLOCK_TIME, + changes: bodies.map((body, index) => ({ ...body, txIndex: 0, msgIndex: index })) + }; + } + + function create(input: { deposit?: string; denom?: string; groups?: NormalizedGroup[] }): AkashChangeBody { + return { + kind: "deploymentCreated", + key: KEY, + denom: input.denom ?? "uakt", + deposit: input.deposit ?? "5000000", + depositor: null, + groups: input.groups ?? [group(1, { cpuUnits: 1000, count: 2 })] + }; + } + + function bidCreated(price: string): AkashChangeBody { + return { kind: "bidCreated", key: LEASE_KEY, price, priceDenom: "uakt" }; + } + + function leaseCreated(): AkashChangeBody { + return { kind: "leaseCreated", key: LEASE_KEY }; + } + + function twoGroups(): NormalizedGroup[] { + return [ + group(1, { cpuUnits: 1000, gpuUnits: 0, memoryBytes: 1024, ephemeralStorageBytes: 100, persistentStorageBytes: 50, count: 2 }), + group(2, { cpuUnits: 500, gpuUnits: 1, memoryBytes: 2048, ephemeralStorageBytes: 200, persistentStorageBytes: 0, count: 3 }) + ]; + } + + function group(gseq: number, resource: Partial): NormalizedGroup { + return { + gseq, + resources: [ + { + count: 1, + cpuUnits: 0, + gpuUnits: 0, + gpuVendor: null, + gpuModel: null, + memoryBytes: 0, + ephemeralStorageBytes: 0, + persistentStorageBytes: 0, + price: "1", + priceDenom: "uakt", + ...resource + } + ] + }; + } +}); diff --git a/apps/chain-indexer/src/akash/network-delta.ts b/apps/chain-indexer/src/akash/network-delta.ts new file mode 100644 index 0000000000..005ee75244 --- /dev/null +++ b/apps/chain-indexer/src/akash/network-delta.ts @@ -0,0 +1,146 @@ +import type { AkashBlockChanges } from "@src/akash/akash-changes"; +import { isProviderChange } from "@src/akash/akash-changes"; +import type { DeploymentAggState } from "@src/akash/deployment-reducer"; +import { stateKey } from "@src/akash/deployment-reducer"; + +/** + * One block's contribution to the network aggregates, measured as the change in the deployment + * states across the reducer fold. `earnedDeltaByDenom` holds 18-decimal Dec atomics of the change + * in cumulative provider earnings — Σ(withdrawn + balance) over leases — which is settlement-exact + * and excludes the close-time fractional refund to the owner by construction. + */ +export interface NetworkBlockDelta { + height: number; + leasesCreated: number; + activeLeaseDelta: number; + cpuUnitsDelta: number; + gpuUnitsDelta: number; + memoryBytesDelta: number; + ephemeralStorageBytesDelta: number; + persistentStorageBytesDelta: number; + earnedDeltaByDenom: Map; +} + +export interface DeploymentNetworkSnapshot { + leaseCount: number; + activeLeaseCount: number; + cpuUnits: number; + gpuUnits: number; + memoryBytes: number; + ephemeralStorageBytes: number; + persistentStorageBytes: number; + earnedByDenom: Map; +} + +/** + * Captures the pre-apply aggregate contribution of every deployment the block's changes reference. + * A deployment the reducer has not loaded (it is created by this block) snapshots as empty, so its + * whole post-apply state counts as delta. + */ +export function snapshotNetworkState(states: Map, block: AkashBlockChanges): Map { + const snapshots = new Map(); + for (const change of block.changes) { + if (isProviderChange(change)) { + continue; + } + const key = stateKey(change.key); + if (!snapshots.has(key)) { + snapshots.set(key, summarizeDeployment(states.get(key))); + } + } + return snapshots; +} + +export function diffNetworkDelta( + before: Map, + states: Map, + block: AkashBlockChanges +): NetworkBlockDelta { + const delta: NetworkBlockDelta = { + height: block.height, + leasesCreated: 0, + activeLeaseDelta: 0, + cpuUnitsDelta: 0, + gpuUnitsDelta: 0, + memoryBytesDelta: 0, + ephemeralStorageBytesDelta: 0, + persistentStorageBytesDelta: 0, + earnedDeltaByDenom: new Map() + }; + + for (const [key, prior] of before) { + const current = summarizeDeployment(states.get(key)); + delta.leasesCreated += current.leaseCount - prior.leaseCount; + delta.activeLeaseDelta += current.activeLeaseCount - prior.activeLeaseCount; + delta.cpuUnitsDelta += current.cpuUnits - prior.cpuUnits; + delta.gpuUnitsDelta += current.gpuUnits - prior.gpuUnits; + delta.memoryBytesDelta += current.memoryBytes - prior.memoryBytes; + delta.ephemeralStorageBytesDelta += current.ephemeralStorageBytes - prior.ephemeralStorageBytes; + delta.persistentStorageBytesDelta += current.persistentStorageBytes - prior.persistentStorageBytes; + accumulateEarnedDelta(delta.earnedDeltaByDenom, current.earnedByDenom, prior.earnedByDenom); + } + + return delta; +} + +export function isEmptyNetworkDelta(delta: NetworkBlockDelta): boolean { + return ( + delta.leasesCreated === 0 && + delta.activeLeaseDelta === 0 && + delta.cpuUnitsDelta === 0 && + delta.gpuUnitsDelta === 0 && + delta.memoryBytesDelta === 0 && + delta.ephemeralStorageBytesDelta === 0 && + delta.persistentStorageBytesDelta === 0 && + delta.earnedDeltaByDenom.size === 0 + ); +} + +function summarizeDeployment(state: DeploymentAggState | undefined): DeploymentNetworkSnapshot { + const snapshot: DeploymentNetworkSnapshot = { + leaseCount: 0, + activeLeaseCount: 0, + cpuUnits: 0, + gpuUnits: 0, + memoryBytes: 0, + ephemeralStorageBytes: 0, + persistentStorageBytes: 0, + earnedByDenom: new Map() + }; + if (!state) { + return snapshot; + } + + snapshot.leaseCount = state.leases.length; + for (const lease of state.leases) { + const earned = lease.withdrawn + lease.balance; + if (earned !== 0n) { + snapshot.earnedByDenom.set(lease.denom, (snapshot.earnedByDenom.get(lease.denom) ?? 0n) + earned); + } + if (lease.closedHeight !== null) { + continue; + } + snapshot.activeLeaseCount += 1; + snapshot.cpuUnits += lease.cpuUnits; + snapshot.gpuUnits += lease.gpuUnits; + snapshot.memoryBytes += lease.memoryBytes; + snapshot.ephemeralStorageBytes += lease.ephemeralStorageBytes; + snapshot.persistentStorageBytes += lease.persistentStorageBytes; + } + return snapshot; +} + +function accumulateEarnedDelta(target: Map, current: Map, prior: Map): void { + for (const denom of new Set([...current.keys(), ...prior.keys()])) { + const change = (current.get(denom) ?? 0n) - (prior.get(denom) ?? 0n); + if (change === 0n) { + continue; + } + const next = (target.get(denom) ?? 0n) + change; + if (next === 0n) { + target.delete(denom); + } else { + target.set(denom, next); + } + } +} diff --git a/apps/chain-indexer/src/db/schema.ts b/apps/chain-indexer/src/db/schema.ts index 815cf23e22..254ab8d012 100644 --- a/apps/chain-indexer/src/db/schema.ts +++ b/apps/chain-indexer/src/db/schema.ts @@ -3,6 +3,8 @@ import { bigint, bigserial, boolean, + check, + date, index, integer, jsonb, @@ -645,3 +647,79 @@ export const BmeCanceledRecords = akashSchema.table( }, t => [primaryKey({ columns: [t.recordHeight, t.sequence, t.denom, t.toDenom, t.source] }), index("bme_canceled_records_height_idx").on(t.height)] ); + +/** + * Singleton current network state, maintained incrementally inside the per-block transaction so + * dashboards read one row instead of scanning leases. `last_aggregated_height` is the replay + * watermark: blocks at or below it are duplicates and must not be re-applied. Spend totals are + * settlement-exact cumulative provider earnings — Σ(withdrawn + balance) over all leases per denom — + * not the legacy rate×blocks estimate, so they reconcile exactly against `akash.leases`. + */ +export const NetworkState = akashSchema.table( + "network_state", + { + id: integer("id").primaryKey(), + lastAggregatedHeight: bigint("last_aggregated_height", { mode: "number" }).notNull(), + lastAggregatedAt: timestamp("last_aggregated_at", { withTimezone: true }).notNull(), + activeLeaseCount: integer("active_lease_count").notNull().default(0), + totalLeaseCount: bigint("total_lease_count", { mode: "number" }).notNull().default(0), + activeProviderCount: integer("active_provider_count").notNull().default(0), + activeCpuUnits: bigint("active_cpu_units", { mode: "number" }).notNull().default(0), + activeGpuUnits: bigint("active_gpu_units", { mode: "number" }).notNull().default(0), + activeMemoryBytes: bigint("active_memory_bytes", { mode: "number" }).notNull().default(0), + activeEphemeralStorageBytes: bigint("active_ephemeral_storage_bytes", { mode: "number" }).notNull().default(0), + activePersistentStorageBytes: bigint("active_persistent_storage_bytes", { mode: "number" }).notNull().default(0), + totalUaktSpent: numeric("total_uakt_spent", { precision: 38, scale: 18 }).notNull().default("0"), + totalUusdcSpent: numeric("total_uusdc_spent", { precision: 38, scale: 18 }).notNull().default("0"), + totalUactSpent: numeric("total_uact_spent", { precision: 38, scale: 18 }).notNull().default("0") + }, + t => [check("network_state_singleton_check", sql`${t.id} = 1`)] +); + +/** + * Append-only daily network rollup, one row per UTC day that had blocks, written atomically with the + * first block of the following day. Rows carry both close-of-day cumulatives and per-day deltas in + * native denoms; `daily_usd_spent` is filled at day close (or restated later) from `daily_prices`, + * with `akt_price_used` marking which price the USD was computed with — a price restatement therefore + * updates exactly the affected day's row. Cumulative USD is deliberately not stored: it is a window + * SUM over this table on read, so restatements never cascade. + */ +export const NetworkRollups = akashSchema.table("network_rollups", { + date: date("date", { mode: "string" }).primaryKey(), + closeHeight: bigint("close_height", { mode: "number" }).notNull(), + closeAt: timestamp("close_at", { withTimezone: true }).notNull(), + activeLeaseCount: integer("active_lease_count").notNull(), + totalLeaseCount: bigint("total_lease_count", { mode: "number" }).notNull(), + dailyLeaseCount: integer("daily_lease_count").notNull(), + activeProviderCount: integer("active_provider_count").notNull(), + activeCpuUnits: bigint("active_cpu_units", { mode: "number" }).notNull(), + activeGpuUnits: bigint("active_gpu_units", { mode: "number" }).notNull(), + activeMemoryBytes: bigint("active_memory_bytes", { mode: "number" }).notNull(), + activeEphemeralStorageBytes: bigint("active_ephemeral_storage_bytes", { mode: "number" }).notNull(), + activePersistentStorageBytes: bigint("active_persistent_storage_bytes", { mode: "number" }).notNull(), + totalUaktSpent: numeric("total_uakt_spent", { precision: 38, scale: 18 }).notNull(), + totalUusdcSpent: numeric("total_uusdc_spent", { precision: 38, scale: 18 }).notNull(), + totalUactSpent: numeric("total_uact_spent", { precision: 38, scale: 18 }).notNull(), + dailyUaktSpent: numeric("daily_uakt_spent", { precision: 38, scale: 18 }).notNull(), + dailyUusdcSpent: numeric("daily_uusdc_spent", { precision: 38, scale: 18 }).notNull(), + dailyUactSpent: numeric("daily_uact_spent", { precision: 38, scale: 18 }).notNull(), + dailyUsdSpent: numeric("daily_usd_spent", { precision: 38, scale: 18 }), + aktPriceUsed: numeric("akt_price_used", { precision: 38, scale: 18 }), + usdComputedAt: timestamp("usd_computed_at", { withTimezone: true }) +}); + +/** + * Daily close prices in USD per whole token, populated by the jobs role (CON-807). Rollup USD values + * are recomputed for exactly the days whose price differs from the `akt_price_used` they were + * computed with, so this table is the single source of truth for restatements. + */ +export const DailyPrices = akashSchema.table( + "daily_prices", + { + date: date("date", { mode: "string" }).notNull(), + denom: text("denom").notNull(), + price: numeric("price", { precision: 38, scale: 18 }).notNull(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow() + }, + t => [primaryKey({ columns: [t.date, t.denom] })] +); diff --git a/apps/chain-indexer/src/network/day-close-usd.service.spec.ts b/apps/chain-indexer/src/network/day-close-usd.service.spec.ts new file mode 100644 index 0000000000..71879aad1d --- /dev/null +++ b/apps/chain-indexer/src/network/day-close-usd.service.spec.ts @@ -0,0 +1,85 @@ +import type { SQL } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { describe, expect, it } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import { DailyPrices } from "@src/db/schema"; +import { DayCloseUsdService } from "@src/network/day-close-usd.service"; +import type { ChainDatabase } from "@src/providers/db.provider"; +import type { LoggerService } from "@src/providers/logging.provider"; + +describe(DayCloseUsdService.name, () => { + it("updates only rollups whose stored price differs from the daily price", async () => { + const { service, executor, captured } = setup({ updatedRows: [{ date: "2026-08-14" }, { date: "2026-08-13" }] }); + + const dates = await service.recompute(executor); + + expect(dates).toEqual(["2026-08-13", "2026-08-14"]); + expect(captured.from).toBe(DailyPrices); + expect(renderSql(captured.where as SQL)).toContain('"akash"."daily_prices"."denom" = '); + expect(renderSql(captured.where as SQL)).toContain('"akash"."daily_prices"."date" = "akash"."network_rollups"."date"'); + expect(renderSql(captured.where as SQL)).toContain('"akash"."network_rollups"."akt_price_used" IS DISTINCT FROM "akash"."daily_prices"."price"'); + }); + + it("converts micro-denom daily spend into whole USD at the day price", async () => { + const { service, executor, captured } = setup({}); + + await service.recompute(executor); + + const set = captured.set as Record; + expect(renderSql(set.dailyUsdSpent as SQL)).toBe( + '"akash"."network_rollups"."daily_uakt_spent" / 1000000::numeric * "akash"."daily_prices"."price" + ("akash"."network_rollups"."daily_uusdc_spent" + "akash"."network_rollups"."daily_uact_spent") / 1000000::numeric' + ); + expect(renderSql(set.aktPriceUsed as SQL)).toBe('"akash"."daily_prices"."price"'); + expect(set.usdComputedAt).toBeInstanceOf(Date); + }); + + it("scopes the restatement to a single day when a date is given", async () => { + const { service, executor, captured } = setup({ updatedRows: [{ date: "2026-08-13" }] }); + + const dates = await service.recompute(executor, "2026-08-13"); + + expect(dates).toEqual(["2026-08-13"]); + expect(renderSql(captured.where as SQL)).toContain('"akash"."network_rollups"."date" = '); + }); + + it("logs nothing when no day needed restating", async () => { + const { service, executor, logger } = setup({ updatedRows: [] }); + + const dates = await service.recompute(executor); + + expect(dates).toEqual([]); + expect(logger.info).not.toHaveBeenCalled(); + }); + + function setup(input: { updatedRows?: { date: string }[] }) { + const captured: { set?: unknown; from?: unknown; where?: unknown } = {}; + + const executor = { + update: () => ({ + set: (values: unknown) => { + captured.set = values; + return { + from: (table: unknown) => { + captured.from = table; + return { + where: (condition: unknown) => { + captured.where = condition; + return { returning: () => Promise.resolve(input.updatedRows ?? []) }; + } + }; + } + }; + } + }) + }; + + const logger = mock(); + const service = new DayCloseUsdService(logger); + return { service, executor: executor as unknown as ChainDatabase, captured, logger }; + } + + function renderSql(fragment: SQL): string { + return new PgDialect().sqlToQuery(fragment).sql; + } +}); diff --git a/apps/chain-indexer/src/network/day-close-usd.service.ts b/apps/chain-indexer/src/network/day-close-usd.service.ts new file mode 100644 index 0000000000..6792d7ecb4 --- /dev/null +++ b/apps/chain-indexer/src/network/day-close-usd.service.ts @@ -0,0 +1,50 @@ +import { and, eq, sql } from "drizzle-orm"; +import { inject, singleton } from "tsyringe"; + +import { DailyPrices, NetworkRollups } from "@src/db/schema"; +import type { ChainDatabase, ChainTransaction } from "@src/providers/db.provider"; +import { LoggerService } from "@src/providers/logging.provider"; + +const MICRO_UNITS_PER_TOKEN = sql`1000000::numeric`; + +/** + * Fills or restates `daily_usd_spent` on the network rollups from `daily_prices`, touching only the + * days whose stored `akt_price_used` differs from the current price — so a restatement updates + * exactly one row per affected day and a rerun is a no-op. uusdc and uact are stablecoins pegged + * 1 USD per whole token, mirroring the legacy USD computation; uakt converts at the day's close price. + */ +@singleton() +export class DayCloseUsdService { + readonly #logger: LoggerService; + + constructor(@inject(LoggerService) logger: LoggerService) { + this.#logger = logger; + this.#logger.setContext("DAY_CLOSE_USD"); + } + + async recompute(executor: ChainDatabase | ChainTransaction, date?: string): Promise { + const updated = await executor + .update(NetworkRollups) + .set({ + dailyUsdSpent: sql`${NetworkRollups.dailyUaktSpent} / ${MICRO_UNITS_PER_TOKEN} * ${DailyPrices.price} + (${NetworkRollups.dailyUusdcSpent} + ${NetworkRollups.dailyUactSpent}) / ${MICRO_UNITS_PER_TOKEN}`, + aktPriceUsed: sql`${DailyPrices.price}`, + usdComputedAt: new Date() + }) + .from(DailyPrices) + .where( + and( + eq(DailyPrices.denom, "uakt"), + eq(DailyPrices.date, NetworkRollups.date), + sql`${NetworkRollups.aktPriceUsed} IS DISTINCT FROM ${DailyPrices.price}`, + date === undefined ? undefined : eq(NetworkRollups.date, date) + ) + ) + .returning({ date: NetworkRollups.date }); + + const dates = updated.map(row => row.date).sort(); + if (dates.length > 0) { + this.#logger.info({ event: "DAILY_USD_RESTATED", count: dates.length, dates }); + } + return dates; + } +} diff --git a/apps/chain-indexer/src/network/network-stats-writer.service.spec.ts b/apps/chain-indexer/src/network/network-stats-writer.service.spec.ts new file mode 100644 index 0000000000..266da87149 --- /dev/null +++ b/apps/chain-indexer/src/network/network-stats-writer.service.spec.ts @@ -0,0 +1,305 @@ +import { describe, expect, it } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import { decFromInt, decToString } from "@src/akash/dec"; +import type { NetworkBlockDelta } from "@src/akash/network-delta"; +import { NetworkRollups, NetworkState } from "@src/db/schema"; +import type { DayCloseUsdService } from "@src/network/day-close-usd.service"; +import { NetworkStatsWriter } from "@src/network/network-stats-writer.service"; +import type { ChainTransaction } from "@src/providers/db.provider"; +import type { LoggerService } from "@src/providers/logging.provider"; + +describe(NetworkStatsWriter.name, () => { + it("lazily initializes the singleton row with the watermark just below the first block", async () => { + const { writer, tx, inserts, updates } = setup(); + + await writer.write(tx, [block(100, "2026-08-13T10:00:00Z")], []); + + expect(rowsFor(inserts, NetworkState)).toEqual([ + expect.objectContaining({ id: 1, lastAggregatedHeight: 99, lastAggregatedAt: new Date("2026-08-13T10:00:00Z") }) + ]); + expect(updates).toEqual([ + expect.objectContaining({ lastAggregatedHeight: 100, lastAggregatedAt: new Date("2026-08-13T10:00:00Z"), activeProviderCount: 5 }) + ]); + }); + + it("applies block deltas onto the running state", async () => { + const { writer, tx, updates } = setup({ stateRow: stateRow({ lastAggregatedHeight: 100 }) }); + + await writer.write( + tx, + [block(101, "2026-08-13T10:00:00Z")], + [ + delta(101, { + leasesCreated: 2, + activeLeaseDelta: 1, + cpuUnitsDelta: 2000, + gpuUnitsDelta: 1, + memoryBytesDelta: 1024, + ephemeralStorageBytesDelta: 100, + persistentStorageBytesDelta: 50, + earnedDeltaByDenom: new Map([ + ["uakt", decFromInt(900)], + ["uusdc", decFromInt(30)] + ]) + }) + ] + ); + + expect(updates).toEqual([ + { + lastAggregatedHeight: 101, + lastAggregatedAt: new Date("2026-08-13T10:00:00Z"), + activeLeaseCount: 4, + totalLeaseCount: 12, + activeProviderCount: 5, + activeCpuUnits: 12000, + activeGpuUnits: 3, + activeMemoryBytes: 11024, + activeEphemeralStorageBytes: 1100, + activePersistentStorageBytes: 1050, + totalUaktSpent: decToString(decFromInt(1900)), + totalUusdcSpent: decToString(decFromInt(30)), + totalUactSpent: decToString(0n) + } + ]); + }); + + it("warns and drops earnings in an unknown denom", async () => { + const { writer, tx, updates, logger } = setup({ stateRow: stateRow({ lastAggregatedHeight: 100 }) }); + + await writer.write(tx, [block(101, "2026-08-13T10:00:00Z")], [delta(101, { earnedDeltaByDenom: new Map([["ibc/deadbeef", decFromInt(5)]]) })]); + + expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ event: "NETWORK_STATS_UNKNOWN_SPEND_DENOM", denom: "ibc/deadbeef" })); + expect(updates[0]).toMatchObject({ totalUaktSpent: decToString(decFromInt(1000)) }); + }); + + it("closes the previous day when a batch crosses a UTC boundary", async () => { + const { writer, tx, inserts, dayCloseUsd } = setup({ stateRow: stateRow({ lastAggregatedHeight: 100 }), providerCounts: [4, 5] }); + + await writer.write( + tx, + [block(101, "2026-08-13T23:59:55Z"), block(102, "2026-08-14T00:00:05Z")], + [delta(101, { leasesCreated: 1, activeLeaseDelta: 1, earnedDeltaByDenom: new Map([["uakt", decFromInt(100)]]) })] + ); + + expect(rowsFor(inserts, NetworkRollups)).toEqual([ + { + date: "2026-08-13", + closeHeight: 101, + closeAt: new Date("2026-08-13T23:59:55Z"), + activeLeaseCount: 4, + totalLeaseCount: 11, + dailyLeaseCount: 11, + activeProviderCount: 4, + activeCpuUnits: 10000, + activeGpuUnits: 2, + activeMemoryBytes: 10000, + activeEphemeralStorageBytes: 1000, + activePersistentStorageBytes: 1000, + totalUaktSpent: decToString(decFromInt(1100)), + totalUusdcSpent: decToString(0n), + totalUactSpent: decToString(0n), + dailyUaktSpent: decToString(decFromInt(1100)), + dailyUusdcSpent: decToString(0n), + dailyUactSpent: decToString(0n) + } + ]); + expect(dayCloseUsd.recompute).toHaveBeenCalledWith(tx, "2026-08-13"); + }); + + it("closes a day whose last block belongs to a previous batch", async () => { + const { writer, tx, inserts } = setup({ + stateRow: stateRow({ lastAggregatedHeight: 100, lastAggregatedAt: new Date("2026-08-13T23:59:55Z") }) + }); + + await writer.write(tx, [block(101, "2026-08-14T00:00:05Z")], []); + + expect(rowsFor(inserts, NetworkRollups)).toEqual([ + expect.objectContaining({ date: "2026-08-13", closeHeight: 100, closeAt: new Date("2026-08-13T23:59:55Z") }) + ]); + }); + + it("computes daily deltas against the previous rollup row", async () => { + const { writer, tx, inserts } = setup({ + stateRow: stateRow({ lastAggregatedHeight: 100, lastAggregatedAt: new Date("2026-08-13T23:59:55Z") }), + rollups: [rollupRow({ date: "2026-08-12", totalLeaseCount: 8, totalUaktSpent: decToString(decFromInt(400)) })] + }); + + await writer.write(tx, [block(101, "2026-08-14T00:00:05Z")], []); + + expect(rowsFor(inserts, NetworkRollups)).toEqual([ + expect.objectContaining({ + date: "2026-08-13", + dailyLeaseCount: 2, + dailyUaktSpent: decToString(decFromInt(600)), + totalUaktSpent: decToString(decFromInt(1000)) + }) + ]); + }); + + it("closes each crossed day once and only for days that had blocks", async () => { + const { writer, tx, inserts } = setup({ stateRow: stateRow({ lastAggregatedHeight: 100 }), providerCounts: [5, 5, 5] }); + + await writer.write(tx, [block(101, "2026-08-13T23:59:55Z"), block(102, "2026-08-16T00:00:05Z"), block(103, "2026-08-17T00:00:05Z")], []); + + expect(rowsFor(inserts, NetworkRollups).map(row => [row.date, row.closeHeight])).toEqual([ + ["2026-08-13", 101], + ["2026-08-16", 102] + ]); + }); + + it("does nothing beyond locking when every block is at or below the watermark", async () => { + const { writer, tx, inserts, updates, dayCloseUsd } = setup({ stateRow: stateRow({ lastAggregatedHeight: 200 }) }); + + await writer.write(tx, [block(101, "2026-08-14T10:00:00Z")], [delta(101, { activeLeaseDelta: 1 })]); + + expect(rowsFor(inserts, NetworkRollups)).toEqual([]); + expect(updates).toEqual([]); + expect(dayCloseUsd.recompute).not.toHaveBeenCalled(); + }); + + function setup(input?: { stateRow?: Record; rollups?: Record[]; providerCounts?: number[] }) { + const inserts: { table: unknown; rows: Record[] }[] = []; + const updates: Record[] = []; + const stateRows: Record[] = input?.stateRow ? [input.stateRow] : []; + const rollupRows: Record[] = [...(input?.rollups ?? [])]; + const providerCounts = [...(input?.providerCounts ?? [])]; + + const rowsFromTable = (table: unknown) => { + if (table === NetworkState) { + return stateRows; + } + if (table === NetworkRollups) { + return [...rollupRows].sort((a, b) => String(b.date).localeCompare(String(a.date))); + } + return [{ value: providerCounts.shift() ?? 5 }]; + }; + + const selectChain = (table: unknown) => { + const chain = { + where: () => chain, + orderBy: () => chain, + limit: () => chain, + for: () => chain, + then: (resolve: (rows: unknown[]) => unknown, reject?: (error: unknown) => unknown) => Promise.resolve(rowsFromTable(table)).then(resolve, reject) + }; + return chain; + }; + + const tx = { + insert: (table: unknown) => ({ + values: (rows: Record | Record[]) => { + const rowArray = Array.isArray(rows) ? rows : [rows]; + inserts.push({ table, rows: rowArray }); + if (table === NetworkState && stateRows.length === 0) { + stateRows.push({ ...emptyStateRow(), ...rowArray[0] }); + } + if (table === NetworkRollups) { + rollupRows.push(...rowArray); + } + return { onConflictDoNothing: () => Promise.resolve() }; + } + }), + select: () => ({ from: (table: unknown) => selectChain(table) }), + update: (table: unknown) => ({ + set: (values: Record) => { + void table; + updates.push(values); + return { where: () => Promise.resolve() }; + } + }) + }; + + const dayCloseUsd = mock(); + const logger = mock(); + const writer = new NetworkStatsWriter(dayCloseUsd, logger); + return { writer, tx: tx as unknown as ChainTransaction, inserts, updates, dayCloseUsd, logger }; + } + + function block(height: number, datetime: string) { + return { height, datetime: new Date(datetime) }; + } + + function delta(height: number, overrides: Partial): NetworkBlockDelta { + return { + height, + leasesCreated: 0, + activeLeaseDelta: 0, + cpuUnitsDelta: 0, + gpuUnitsDelta: 0, + memoryBytesDelta: 0, + ephemeralStorageBytesDelta: 0, + persistentStorageBytesDelta: 0, + earnedDeltaByDenom: new Map(), + ...overrides + }; + } + + function stateRow(overrides: Record) { + return { + id: 1, + lastAggregatedHeight: 100, + lastAggregatedAt: new Date("2026-08-13T09:00:00Z"), + activeLeaseCount: 3, + totalLeaseCount: 10, + activeProviderCount: 5, + activeCpuUnits: 10000, + activeGpuUnits: 2, + activeMemoryBytes: 10000, + activeEphemeralStorageBytes: 1000, + activePersistentStorageBytes: 1000, + totalUaktSpent: decToString(decFromInt(1000)), + totalUusdcSpent: decToString(0n), + totalUactSpent: decToString(0n), + ...overrides + }; + } + + function rollupRow(overrides: Record) { + return { + date: "2026-08-12", + closeHeight: 90, + closeAt: new Date("2026-08-12T23:59:00Z"), + activeLeaseCount: 2, + totalLeaseCount: 8, + dailyLeaseCount: 8, + activeProviderCount: 4, + activeCpuUnits: 8000, + activeGpuUnits: 1, + activeMemoryBytes: 8000, + activeEphemeralStorageBytes: 800, + activePersistentStorageBytes: 800, + totalUaktSpent: decToString(decFromInt(400)), + totalUusdcSpent: decToString(0n), + totalUactSpent: decToString(0n), + dailyUaktSpent: decToString(decFromInt(400)), + dailyUusdcSpent: decToString(0n), + dailyUactSpent: decToString(0n), + dailyUsdSpent: null, + aktPriceUsed: null, + usdComputedAt: null, + ...overrides + }; + } + + function emptyStateRow() { + return { + activeLeaseCount: 0, + totalLeaseCount: 0, + activeProviderCount: 0, + activeCpuUnits: 0, + activeGpuUnits: 0, + activeMemoryBytes: 0, + activeEphemeralStorageBytes: 0, + activePersistentStorageBytes: 0, + totalUaktSpent: "0", + totalUusdcSpent: "0", + totalUactSpent: "0" + }; + } + + function rowsFor(inserts: { table: unknown; rows: Record[] }[], table: unknown): Record[] { + return inserts.filter(insert => insert.table === table).flatMap(insert => insert.rows); + } +}); diff --git a/apps/chain-indexer/src/network/network-stats-writer.service.ts b/apps/chain-indexer/src/network/network-stats-writer.service.ts new file mode 100644 index 0000000000..0ad9dc56c7 --- /dev/null +++ b/apps/chain-indexer/src/network/network-stats-writer.service.ts @@ -0,0 +1,208 @@ +import { and, count, desc, eq, isNull, lte } from "drizzle-orm"; +import { inject, singleton } from "tsyringe"; + +import { decFromString, decToString } from "@src/akash/dec"; +import type { NetworkBlockDelta } from "@src/akash/network-delta"; +import { NetworkRollups, NetworkState, Providers } from "@src/db/schema"; +import { DayCloseUsdService } from "@src/network/day-close-usd.service"; +import type { ChainTransaction } from "@src/providers/db.provider"; +import { LoggerService } from "@src/providers/logging.provider"; + +export interface NetworkBlockRef { + height: number; + datetime: Date; +} + +interface RunningNetworkState { + lastAggregatedHeight: number; + lastAggregatedAt: Date; + activeLeaseCount: number; + totalLeaseCount: number; + activeProviderCount: number; + activeCpuUnits: number; + activeGpuUnits: number; + activeMemoryBytes: number; + activeEphemeralStorageBytes: number; + activePersistentStorageBytes: number; + totalUaktSpent: bigint; + totalUusdcSpent: bigint; + totalUactSpent: bigint; +} + +const SINGLETON_ID = 1; + +const SPENT_FIELD_BY_DENOM = { + uakt: "totalUaktSpent", + uusdc: "totalUusdcSpent", + uact: "totalUactSpent" +} as const; + +function utcDay(datetime: Date): string { + return datetime.toISOString().slice(0, 10); +} + +/** + * Maintains the singleton current-network-state row and the append-only daily rollups inside the + * block transaction, from the per-block deltas the akash writer measured across its reducer fold. + * The row is locked FOR UPDATE and `last_aggregated_height` is the replay watermark, so duplicate + * commits and overlapping writers fold each block exactly once. A UTC day is closed atomically with + * the first block of the following day; the closed day's USD is filled immediately when its price is + * already known (backfill), and otherwise later by the price job via DayCloseUsdService. + */ +@singleton() +export class NetworkStatsWriter { + readonly #dayCloseUsd: DayCloseUsdService; + readonly #logger: LoggerService; + + constructor(@inject(DayCloseUsdService) dayCloseUsd: DayCloseUsdService, @inject(LoggerService) logger: LoggerService) { + this.#dayCloseUsd = dayCloseUsd; + this.#logger = logger; + this.#logger.setContext("NETWORK_STATS"); + } + + async write(tx: ChainTransaction, blocks: NetworkBlockRef[], deltas: NetworkBlockDelta[]): Promise { + if (blocks.length === 0) { + return; + } + + await this.#ensureStateRow(tx, blocks[0]); + const state = await this.#lockState(tx); + const deltaByHeight = new Map(deltas.map(delta => [delta.height, delta])); + + let aggregated = false; + for (const block of blocks) { + if (block.height <= state.lastAggregatedHeight) { + continue; + } + if (utcDay(block.datetime) > utcDay(state.lastAggregatedAt)) { + await this.#closeDay(tx, state); + } + const delta = deltaByHeight.get(block.height); + if (delta) { + this.#applyDelta(state, delta); + } + state.lastAggregatedHeight = block.height; + state.lastAggregatedAt = block.datetime; + aggregated = true; + } + + if (!aggregated) { + return; + } + + state.activeProviderCount = await this.#countProviders(tx); + await this.#flushState(tx, state); + } + + async #ensureStateRow(tx: ChainTransaction, firstBlock: NetworkBlockRef): Promise { + await tx + .insert(NetworkState) + .values({ id: SINGLETON_ID, lastAggregatedHeight: firstBlock.height - 1, lastAggregatedAt: firstBlock.datetime }) + .onConflictDoNothing(); + } + + async #lockState(tx: ChainTransaction): Promise { + const [row] = await tx.select().from(NetworkState).where(eq(NetworkState.id, SINGLETON_ID)).for("update"); + return { + lastAggregatedHeight: row.lastAggregatedHeight, + lastAggregatedAt: row.lastAggregatedAt, + activeLeaseCount: row.activeLeaseCount, + totalLeaseCount: row.totalLeaseCount, + activeProviderCount: row.activeProviderCount, + activeCpuUnits: row.activeCpuUnits, + activeGpuUnits: row.activeGpuUnits, + activeMemoryBytes: row.activeMemoryBytes, + activeEphemeralStorageBytes: row.activeEphemeralStorageBytes, + activePersistentStorageBytes: row.activePersistentStorageBytes, + totalUaktSpent: decFromString(row.totalUaktSpent), + totalUusdcSpent: decFromString(row.totalUusdcSpent), + totalUactSpent: decFromString(row.totalUactSpent) + }; + } + + #applyDelta(state: RunningNetworkState, delta: NetworkBlockDelta): void { + state.activeLeaseCount += delta.activeLeaseDelta; + state.totalLeaseCount += delta.leasesCreated; + state.activeCpuUnits += delta.cpuUnitsDelta; + state.activeGpuUnits += delta.gpuUnitsDelta; + state.activeMemoryBytes += delta.memoryBytesDelta; + state.activeEphemeralStorageBytes += delta.ephemeralStorageBytesDelta; + state.activePersistentStorageBytes += delta.persistentStorageBytesDelta; + + for (const [denom, earned] of delta.earnedDeltaByDenom) { + const field = SPENT_FIELD_BY_DENOM[denom as keyof typeof SPENT_FIELD_BY_DENOM]; + if (!field) { + this.#logger.warn({ event: "NETWORK_STATS_UNKNOWN_SPEND_DENOM", denom, height: delta.height, earned: decToString(earned) }); + continue; + } + state[field] += earned; + } + } + + /** The closing day's snapshot is the state as of the last aggregated block, which may belong to a previous batch. */ + async #closeDay(tx: ChainTransaction, state: RunningNetworkState): Promise { + const date = utcDay(state.lastAggregatedAt); + const [previous] = await tx.select().from(NetworkRollups).orderBy(desc(NetworkRollups.date)).limit(1); + const previousTotals = { + totalLeaseCount: previous?.totalLeaseCount ?? 0, + totalUaktSpent: previous ? decFromString(previous.totalUaktSpent) : 0n, + totalUusdcSpent: previous ? decFromString(previous.totalUusdcSpent) : 0n, + totalUactSpent: previous ? decFromString(previous.totalUactSpent) : 0n + }; + + await tx + .insert(NetworkRollups) + .values({ + date, + closeHeight: state.lastAggregatedHeight, + closeAt: state.lastAggregatedAt, + activeLeaseCount: state.activeLeaseCount, + totalLeaseCount: state.totalLeaseCount, + dailyLeaseCount: state.totalLeaseCount - previousTotals.totalLeaseCount, + activeProviderCount: await this.#countProviders(tx, state.lastAggregatedHeight), + activeCpuUnits: state.activeCpuUnits, + activeGpuUnits: state.activeGpuUnits, + activeMemoryBytes: state.activeMemoryBytes, + activeEphemeralStorageBytes: state.activeEphemeralStorageBytes, + activePersistentStorageBytes: state.activePersistentStorageBytes, + totalUaktSpent: decToString(state.totalUaktSpent), + totalUusdcSpent: decToString(state.totalUusdcSpent), + totalUactSpent: decToString(state.totalUactSpent), + dailyUaktSpent: decToString(state.totalUaktSpent - previousTotals.totalUaktSpent), + dailyUusdcSpent: decToString(state.totalUusdcSpent - previousTotals.totalUusdcSpent), + dailyUactSpent: decToString(state.totalUactSpent - previousTotals.totalUactSpent) + }) + .onConflictDoNothing(); + + await this.#dayCloseUsd.recompute(tx, date); + } + + async #countProviders(tx: ChainTransaction, atHeight?: number): Promise { + const [row] = await tx + .select({ value: count() }) + .from(Providers) + .where(atHeight === undefined ? isNull(Providers.deletedHeight) : and(isNull(Providers.deletedHeight), lte(Providers.createdHeight, atHeight))); + return row.value; + } + + async #flushState(tx: ChainTransaction, state: RunningNetworkState): Promise { + await tx + .update(NetworkState) + .set({ + lastAggregatedHeight: state.lastAggregatedHeight, + lastAggregatedAt: state.lastAggregatedAt, + activeLeaseCount: state.activeLeaseCount, + totalLeaseCount: state.totalLeaseCount, + activeProviderCount: state.activeProviderCount, + activeCpuUnits: state.activeCpuUnits, + activeGpuUnits: state.activeGpuUnits, + activeMemoryBytes: state.activeMemoryBytes, + activeEphemeralStorageBytes: state.activeEphemeralStorageBytes, + activePersistentStorageBytes: state.activePersistentStorageBytes, + totalUaktSpent: decToString(state.totalUaktSpent), + totalUusdcSpent: decToString(state.totalUusdcSpent), + totalUactSpent: decToString(state.totalUactSpent) + }) + .where(eq(NetworkState.id, SINGLETON_ID)); + } +} diff --git a/apps/chain-indexer/src/network/recompute-usd.ts b/apps/chain-indexer/src/network/recompute-usd.ts new file mode 100644 index 0000000000..3fe63a0bd3 --- /dev/null +++ b/apps/chain-indexer/src/network/recompute-usd.ts @@ -0,0 +1,37 @@ +import "@src/providers"; + +import { createOtelLogger } from "@akashnetwork/logging/otel"; +import { container } from "tsyringe"; + +import { envSchema } from "@src/config/env.config"; +import { PgClientService } from "@src/db/pg-client.service"; +import { DayCloseUsdService } from "@src/network/day-close-usd.service"; +import { CHAIN_DB } from "@src/providers/db.provider"; + +/** + * One-shot USD restatement entrypoint (`npm run network:recompute-usd`): recomputes `daily_usd_spent` + * for every rollup day whose stored price differs from `daily_prices`, touching one row per affected + * day. Exits 0 on success (including nothing to restate), non-zero on failure. + */ +async function main(): Promise { + const logger = createOtelLogger({ context: "RECOMPUTE_USD_CLI" }); + + const parsed = envSchema.safeParse(process.env); + if (!parsed.success) { + logger.error({ event: "CONFIG_INVALID", issues: parsed.error.issues.map(issue => ({ path: issue.path.join(".") || "(root)", message: issue.message })) }); + process.exitCode = 1; + return; + } + + try { + const dates = await container.resolve(DayCloseUsdService).recompute(container.resolve(CHAIN_DB)); + logger.info({ event: "RECOMPUTE_USD_DONE", count: dates.length }); + } catch (error) { + logger.error({ event: "RECOMPUTE_USD_FATAL", error }); + process.exitCode = 1; + } finally { + await container.resolve(PgClientService).dispose(); + } +} + +void main(); diff --git a/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts b/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts index d06b7409dd..34f68a2b3c 100644 --- a/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts +++ b/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts @@ -4,10 +4,12 @@ import { describe, expect, it } from "vitest"; import { mock } from "vitest-mock-extended"; import type { AkashWriter } from "@src/akash/akash-writer.service"; +import type { NetworkBlockDelta } from "@src/akash/network-delta"; import type { ProviderWriter } from "@src/akash/provider-writer.service"; import type { BmeWriter } from "@src/bme/bme-writer.service"; import { AccountTxs, Blocks, IndexerState, MessageDeadLetters, Messages, MessageTypes } from "@src/db/schema"; import type { GovWriter } from "@src/gov/gov-writer.service"; +import type { NetworkStatsWriter } from "@src/network/network-stats-writer.service"; import type { AccountInterner } from "@src/pipeline/balance/account-interner.service"; import type { BalanceWriter } from "@src/pipeline/balance/balance-writer.service"; import { BlockCommitterService } from "@src/pipeline/block-committer.service"; @@ -298,6 +300,17 @@ describe(BlockCommitterService.name, () => { expect(accountIds.get("akash1bmeburner")).toBeDefined(); expect(accountIds.get("akash1bmeminter")).toBeDefined(); }); + + it("hands the network stats writer the batch blocks and the akash writer's deltas", async () => { + const { committer, akashWriter, networkStatsWriter } = setup({ selectResults: [[{ id: 7, type: MSG_SEND }]] }); + const deltas = [mock({ height: 10 })]; + akashWriter.write.mockResolvedValue({ networkDeltas: deltas }); + + const block = buildBlock([MSG_SEND], 10); + await committer.commit(block); + + expect(networkStatsWriter.write).toHaveBeenCalledWith(expect.anything(), [block], deltas); + }); }); function setup(input?: { @@ -350,8 +363,10 @@ describe(BlockCommitterService.name, () => { const govWriter = mock(); const akashWriter = mock(); + akashWriter.write.mockResolvedValue({ networkDeltas: [] }); const providerWriter = mock(); const bmeWriter = mock(); + const networkStatsWriter = mock(); const logger = mock(); const committer = new BlockCommitterService( dbFake as unknown as ChainDatabase, @@ -361,9 +376,23 @@ describe(BlockCommitterService.name, () => { akashWriter, providerWriter, bmeWriter, + networkStatsWriter, logger ); - return { committer, insertedRows, conflictUpdates, deletions, interner, balanceWriter, govWriter, akashWriter, providerWriter, bmeWriter, logger }; + return { + committer, + insertedRows, + conflictUpdates, + deletions, + interner, + balanceWriter, + govWriter, + akashWriter, + providerWriter, + bmeWriter, + networkStatsWriter, + logger + }; } function buildBlock( diff --git a/apps/chain-indexer/src/pipeline/block-committer.service.ts b/apps/chain-indexer/src/pipeline/block-committer.service.ts index 8d594c2e8e..da9a856684 100644 --- a/apps/chain-indexer/src/pipeline/block-committer.service.ts +++ b/apps/chain-indexer/src/pipeline/block-committer.service.ts @@ -15,6 +15,7 @@ import { insertChunked } from "@src/db/insert-chunked"; import { AccountTxs, Blocks, IndexerState, MessageDeadLetters, Messages, MessageTypes, Transactions } from "@src/db/schema"; import { sqlExcluded } from "@src/db/sql-excluded"; import { GovWriter } from "@src/gov/gov-writer.service"; +import { NetworkStatsWriter } from "@src/network/network-stats-writer.service"; import { AccountInterner, requireAccountId } from "@src/pipeline/balance/account-interner.service"; import type { DerivedAccountTx } from "@src/pipeline/balance/account-tx-deriver"; import { deriveAccountTxs } from "@src/pipeline/balance/account-tx-deriver"; @@ -53,6 +54,7 @@ export class BlockCommitterService { readonly #akashWriter: AkashWriter; readonly #providerWriter: ProviderWriter; readonly #bmeWriter: BmeWriter; + readonly #networkStatsWriter: NetworkStatsWriter; readonly #logger: LoggerService; readonly #moduleRegistry = buildModuleAddressRegistry(); readonly #typeIds = new Map(); @@ -65,6 +67,7 @@ export class BlockCommitterService { @inject(AkashWriter) akashWriter: AkashWriter, @inject(ProviderWriter) providerWriter: ProviderWriter, @inject(BmeWriter) bmeWriter: BmeWriter, + @inject(NetworkStatsWriter) networkStatsWriter: NetworkStatsWriter, @inject(LoggerService) logger: LoggerService ) { this.#db = db; @@ -74,6 +77,7 @@ export class BlockCommitterService { this.#akashWriter = akashWriter; this.#providerWriter = providerWriter; this.#bmeWriter = bmeWriter; + this.#networkStatsWriter = networkStatsWriter; this.#logger = logger; this.#logger.setContext("COMMITTER"); } @@ -158,9 +162,10 @@ export class BlockCommitterService { await this.#balanceWriter.write(tx, balanceIntents); await insertChunked(tx, AccountTxs, accountTxRows); await this.#govWriter.writeForBlocks(tx, blocks, accountIds); - await this.#akashWriter.write(tx, akashChanges, accountIds); + const { networkDeltas } = await this.#akashWriter.write(tx, akashChanges, accountIds); await this.#providerWriter.write(tx, akashChanges, accountIds); await this.#bmeWriter.write(tx, bmeChanges, accountIds); + await this.#networkStatsWriter.write(tx, blocks, networkDeltas); await tx .insert(IndexerState) diff --git a/apps/chain-indexer/src/reconcile/network-stats-reconciler.spec.ts b/apps/chain-indexer/src/reconcile/network-stats-reconciler.spec.ts new file mode 100644 index 0000000000..dd7a931f96 --- /dev/null +++ b/apps/chain-indexer/src/reconcile/network-stats-reconciler.spec.ts @@ -0,0 +1,207 @@ +import { describe, expect, it } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import { IndexerState, Leases, NetworkRollups, NetworkState } from "@src/db/schema"; +import type { ChainDatabase } from "@src/providers/db.provider"; +import type { LoggerService } from "@src/providers/logging.provider"; +import { NetworkStatsReconciler } from "@src/reconcile/network-stats-reconciler"; + +describe(NetworkStatsReconciler.name, () => { + it("returns true when the state row matches the recomputation from leases", async () => { + const { service, logger } = setup({}); + + await expect(service.reconcile()).resolves.toBe(true); + + expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ event: "NETWORK_RECONCILE_OK", mismatches: 0 })); + expect(logger.error).not.toHaveBeenCalled(); + }); + + it("treats differently formatted but equal numerics as matching", async () => { + const { service } = setup({ + stateRow: stateRow({ totalUaktSpent: "1000.000000000000000000" }), + spendRows: [{ denom: "uakt", earned: "1000" }] + }); + + await expect(service.reconcile()).resolves.toBe(true); + }); + + it("reports a lease count drift on the current state", async () => { + const { service, logger } = setup({ leaseAggregates: [leaseAggregates({ activeLeaseCount: 4 })] }); + + await expect(service.reconcile()).resolves.toBe(false); + + expect(logger.error).toHaveBeenCalledWith( + expect.objectContaining({ event: "NETWORK_RECONCILE_MISMATCH", scope: "network_state", field: "activeLeaseCount", expected: "4", actual: "3" }) + ); + }); + + it("reports a settlement-exact spend drift per denom", async () => { + const { service, logger } = setup({ spendRows: [{ denom: "uakt", earned: "999" }] }); + + await expect(service.reconcile()).resolves.toBe(false); + + expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ event: "NETWORK_RECONCILE_MISMATCH", field: "total_uakt_spent" })); + }); + + it("reports a provider count drift", async () => { + const { service, logger } = setup({ providerCount: 7 }); + + await expect(service.reconcile()).resolves.toBe(false); + + expect(logger.error).toHaveBeenCalledWith( + expect.objectContaining({ event: "NETWORK_RECONCILE_MISMATCH", field: "activeProviderCount", expected: "7", actual: "5" }) + ); + }); + + it("reports a watermark lagging the sync checkpoint", async () => { + const { service, logger } = setup({ checkpoint: 250 }); + + await expect(service.reconcile()).resolves.toBe(false); + + expect(logger.error).toHaveBeenCalledWith( + expect.objectContaining({ event: "NETWORK_RECONCILE_MISMATCH", field: "lastAggregatedHeight", expected: "250", actual: "100" }) + ); + }); + + it("re-derives sampled rollup rows at their close height", async () => { + const { service, logger } = setup({ + rollups: [rollupRow({ date: "2026-08-13", activeGpuUnits: 9 })], + leaseAggregates: [leaseAggregates({}), leaseAggregates({ activeGpuUnits: 2 })] + }); + + await expect(service.reconcile()).resolves.toBe(false); + + expect(logger.error).toHaveBeenCalledWith( + expect.objectContaining({ event: "NETWORK_RECONCILE_MISMATCH", scope: "network_rollups:2026-08-13", field: "activeGpuUnits", expected: "2", actual: "9" }) + ); + }); + + it("passes on an empty database with no state row", async () => { + const { service, logger } = setup({ stateRow: null, leaseAggregates: [leaseAggregates({ activeLeaseCount: 0, totalLeaseCount: 0 })] }); + + await expect(service.reconcile()).resolves.toBe(true); + + expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ event: "NETWORK_RECONCILE_EMPTY" })); + }); + + it("fails when leases exist but the state row is missing", async () => { + const { service, logger } = setup({ stateRow: null }); + + await expect(service.reconcile()).resolves.toBe(false); + + expect(logger.error).toHaveBeenCalledWith(expect.objectContaining({ event: "NETWORK_RECONCILE_NO_STATE" })); + }); + + function setup(input: { + stateRow?: Record | null; + leaseAggregates?: Record[]; + spendRows?: { denom: string; earned: string }[]; + providerCount?: number; + checkpoint?: number; + rollups?: Record[]; + }) { + const leaseAggQueue = [...(input.leaseAggregates ?? [leaseAggregates({})])]; + + const resolveRows = (table: unknown, grouped: boolean): unknown[] => { + if (table === NetworkState) { + return input.stateRow === null ? [] : [input.stateRow ?? stateRow({})]; + } + if (table === IndexerState) { + return [{ stream: "sync", lastHeight: input.checkpoint ?? 100 }]; + } + if (table === NetworkRollups) { + return input.rollups ?? []; + } + if (table === Leases && grouped) { + return input.spendRows ?? [{ denom: "uakt", earned: "1000.000000000000000000" }]; + } + if (table === Leases) { + return [leaseAggQueue.shift() ?? leaseAggregates({})]; + } + return [{ value: input.providerCount ?? 5 }]; + }; + + const makeChain = (table: unknown) => { + let grouped = false; + const chain = { + where: () => chain, + orderBy: () => chain, + limit: () => chain, + groupBy: () => { + grouped = true; + return chain; + }, + then: (resolve: (rows: unknown[]) => unknown, reject?: (error: unknown) => unknown) => + Promise.resolve(resolveRows(table, grouped)).then(resolve, reject) + }; + return chain; + }; + + const tx = { select: () => ({ from: (table: unknown) => makeChain(table) }) }; + const dbFake = { transaction: async (callback: (transaction: unknown) => unknown) => callback(tx) }; + + const logger = mock(); + const service = new NetworkStatsReconciler(dbFake as unknown as ChainDatabase, logger); + return { service, logger }; + } + + function stateRow(overrides: Record) { + return { + id: 1, + lastAggregatedHeight: 100, + lastAggregatedAt: new Date("2026-08-13T10:00:00Z"), + activeLeaseCount: 3, + totalLeaseCount: 10, + activeProviderCount: 5, + activeCpuUnits: 10000, + activeGpuUnits: 2, + activeMemoryBytes: 10000, + activeEphemeralStorageBytes: 1000, + activePersistentStorageBytes: 1000, + totalUaktSpent: "1000.000000000000000000", + totalUusdcSpent: "0.000000000000000000", + totalUactSpent: "0.000000000000000000", + ...overrides + }; + } + + function leaseAggregates(overrides: Record) { + return { + activeLeaseCount: 3, + totalLeaseCount: 10, + activeCpuUnits: 10000, + activeGpuUnits: 2, + activeMemoryBytes: 10000, + activeEphemeralStorageBytes: 1000, + activePersistentStorageBytes: 1000, + ...overrides + }; + } + + function rollupRow(overrides: Record) { + return { + date: "2026-08-13", + closeHeight: 90, + closeAt: new Date("2026-08-13T23:59:00Z"), + activeLeaseCount: 3, + totalLeaseCount: 10, + dailyLeaseCount: 2, + activeProviderCount: 5, + activeCpuUnits: 10000, + activeGpuUnits: 2, + activeMemoryBytes: 10000, + activeEphemeralStorageBytes: 1000, + activePersistentStorageBytes: 1000, + totalUaktSpent: "1000.000000000000000000", + totalUusdcSpent: "0.000000000000000000", + totalUactSpent: "0.000000000000000000", + dailyUaktSpent: "1000.000000000000000000", + dailyUusdcSpent: "0.000000000000000000", + dailyUactSpent: "0.000000000000000000", + dailyUsdSpent: null, + aktPriceUsed: null, + usdComputedAt: null, + ...overrides + }; + } +}); diff --git a/apps/chain-indexer/src/reconcile/network-stats-reconciler.ts b/apps/chain-indexer/src/reconcile/network-stats-reconciler.ts new file mode 100644 index 0000000000..60b48389c5 --- /dev/null +++ b/apps/chain-indexer/src/reconcile/network-stats-reconciler.ts @@ -0,0 +1,174 @@ +import { desc, eq, isNull, sql } from "drizzle-orm"; +import { inject, singleton } from "tsyringe"; + +import { decFromString } from "@src/akash/dec"; +import { IndexerState, Leases, NetworkRollups, NetworkState, Providers } from "@src/db/schema"; +import { SYNC_STREAM } from "@src/pipeline/block-committer.service"; +import type { ChainDatabase, ChainTransaction } from "@src/providers/db.provider"; +import { CHAIN_DB } from "@src/providers/db.provider"; +import { LoggerService } from "@src/providers/logging.provider"; + +const DEFAULT_ROLLUP_SAMPLE_SIZE = 5; + +const TRACKED_SPEND_DENOMS = ["uakt", "uusdc", "uact"] as const; + +interface LeaseAggregates { + activeLeaseCount: number; + totalLeaseCount: number; + activeCpuUnits: number; + activeGpuUnits: number; + activeMemoryBytes: number; + activeEphemeralStorageBytes: number; + activePersistentStorageBytes: number; +} + +interface Mismatch { + scope: string; + field: string; + expected: string; + actual: string; +} + +/** + * Proves the incremental network aggregates match a manual recomputation from the leases table, + * inside one REPEATABLE READ snapshot so the comparison cannot straddle a commit. The current + * `network_state` row is checked in full — counts, active resources, provider count, watermark vs + * the sync checkpoint, and settlement-exact spend (Σ withdrawn + balance per denom). Sampled rollup + * rows are re-derived at their `close_height` from lease created/closed heights; historical spend + * and provider counts are not re-derivable from current rows and are covered by the current-state + * checks instead. + */ +@singleton() +export class NetworkStatsReconciler { + readonly #db: ChainDatabase; + readonly #logger: LoggerService; + + constructor(@inject(CHAIN_DB) db: ChainDatabase, @inject(LoggerService) logger: LoggerService) { + this.#db = db; + this.#logger = logger; + this.#logger.setContext("NETWORK_RECONCILE"); + } + + async reconcile({ rollupSampleSize = DEFAULT_ROLLUP_SAMPLE_SIZE }: { rollupSampleSize?: number } = {}): Promise { + const mismatches = await this.#db.transaction(async tx => this.#collectMismatches(tx, rollupSampleSize), { + isolationLevel: "repeatable read", + accessMode: "read only" + }); + + if (mismatches === undefined) { + return false; + } + + for (const mismatch of mismatches) { + this.#logger.error({ event: "NETWORK_RECONCILE_MISMATCH", ...mismatch }); + } + const ok = mismatches.length === 0; + this.#logger[ok ? "info" : "error"]({ event: ok ? "NETWORK_RECONCILE_OK" : "NETWORK_RECONCILE_FAILED", mismatches: mismatches.length }); + return ok; + } + + async #collectMismatches(tx: ChainTransaction, rollupSampleSize: number): Promise { + const [state] = await tx.select().from(NetworkState); + if (!state) { + const { totalLeaseCount } = await this.#leaseAggregates(tx); + if (totalLeaseCount === 0) { + this.#logger.info({ event: "NETWORK_RECONCILE_EMPTY" }); + return []; + } + this.#logger.error({ event: "NETWORK_RECONCILE_NO_STATE", totalLeaseCount }); + return undefined; + } + + return [...(await this.#checkCurrentState(tx, state)), ...(await this.#checkSpends(tx, state)), ...(await this.#checkRollups(tx, rollupSampleSize))]; + } + + async #checkCurrentState(tx: ChainTransaction, state: typeof NetworkState.$inferSelect): Promise { + const mismatches: Mismatch[] = []; + const recomputed = await this.#leaseAggregates(tx); + + for (const field of Object.keys(recomputed) as (keyof LeaseAggregates)[]) { + if (state[field] !== recomputed[field]) { + mismatches.push({ scope: "network_state", field, expected: String(recomputed[field]), actual: String(state[field]) }); + } + } + + const [providerRow] = await tx + .select({ value: sql`COUNT(*)`.mapWith(Number) }) + .from(Providers) + .where(isNull(Providers.deletedHeight)); + if (state.activeProviderCount !== providerRow.value) { + mismatches.push({ scope: "network_state", field: "activeProviderCount", expected: String(providerRow.value), actual: String(state.activeProviderCount) }); + } + + const [checkpoint] = await tx.select().from(IndexerState).where(eq(IndexerState.stream, SYNC_STREAM)); + if (checkpoint && checkpoint.lastHeight !== state.lastAggregatedHeight) { + mismatches.push({ + scope: "network_state", + field: "lastAggregatedHeight", + expected: String(checkpoint.lastHeight), + actual: String(state.lastAggregatedHeight) + }); + } + + return mismatches; + } + + async #checkSpends(tx: ChainTransaction, state: typeof NetworkState.$inferSelect): Promise { + const rows = await tx + .select({ denom: Leases.denom, earned: sql`SUM(${Leases.withdrawnAmount} + ${Leases.balance})` }) + .from(Leases) + .groupBy(Leases.denom); + const earnedByDenom = new Map(rows.map(row => [row.denom, decFromString(row.earned)])); + + const stateByDenom = { + uakt: decFromString(state.totalUaktSpent), + uusdc: decFromString(state.totalUusdcSpent), + uact: decFromString(state.totalUactSpent) + }; + + const mismatches: Mismatch[] = []; + for (const denom of TRACKED_SPEND_DENOMS) { + const expected = earnedByDenom.get(denom) ?? 0n; + if (stateByDenom[denom] !== expected) { + mismatches.push({ scope: "network_state", field: `total_${denom}_spent`, expected: expected.toString(), actual: stateByDenom[denom].toString() }); + } + } + return mismatches; + } + + async #checkRollups(tx: ChainTransaction, rollupSampleSize: number): Promise { + const rollups = await tx.select().from(NetworkRollups).orderBy(desc(NetworkRollups.date)).limit(rollupSampleSize); + + const mismatches: Mismatch[] = []; + for (const rollup of rollups) { + const recomputed = await this.#leaseAggregates(tx, rollup.closeHeight); + for (const field of Object.keys(recomputed) as (keyof LeaseAggregates)[]) { + if (rollup[field] !== recomputed[field]) { + mismatches.push({ scope: `network_rollups:${rollup.date}`, field, expected: String(recomputed[field]), actual: String(rollup[field]) }); + } + } + } + return mismatches; + } + + async #leaseAggregates(tx: ChainTransaction, atHeight?: number): Promise { + const active = + atHeight === undefined + ? sql`${Leases.closedHeight} IS NULL` + : sql`${Leases.createdHeight} <= ${atHeight} AND (${Leases.closedHeight} IS NULL OR ${Leases.closedHeight} > ${atHeight})`; + const created = atHeight === undefined ? sql`TRUE` : sql`${Leases.createdHeight} <= ${atHeight}`; + + const [row] = await tx + .select({ + activeLeaseCount: sql`COUNT(*) FILTER (WHERE ${active})`.mapWith(Number), + totalLeaseCount: sql`COUNT(*) FILTER (WHERE ${created})`.mapWith(Number), + activeCpuUnits: sql`COALESCE(SUM(${Leases.cpuUnits}) FILTER (WHERE ${active}), 0)`.mapWith(Number), + activeGpuUnits: sql`COALESCE(SUM(${Leases.gpuUnits}) FILTER (WHERE ${active}), 0)`.mapWith(Number), + activeMemoryBytes: sql`COALESCE(SUM(${Leases.memoryBytes}) FILTER (WHERE ${active}), 0)`.mapWith(Number), + activeEphemeralStorageBytes: sql`COALESCE(SUM(${Leases.ephemeralStorageBytes}) FILTER (WHERE ${active}), 0)`.mapWith(Number), + activePersistentStorageBytes: sql`COALESCE(SUM(${Leases.persistentStorageBytes}) FILTER (WHERE ${active}), 0)`.mapWith(Number) + }) + .from(Leases); + return row; + } +} diff --git a/apps/chain-indexer/src/reconcile/reconcile.ts b/apps/chain-indexer/src/reconcile/reconcile.ts index 3a2d9a98f3..8dfeb6f920 100644 --- a/apps/chain-indexer/src/reconcile/reconcile.ts +++ b/apps/chain-indexer/src/reconcile/reconcile.ts @@ -5,11 +5,13 @@ import { container } from "tsyringe"; import { envSchema } from "@src/config/env.config"; import { PgClientService } from "@src/db/pg-client.service"; +import { NetworkStatsReconciler } from "@src/reconcile/network-stats-reconciler"; import { ReconcileService } from "@src/reconcile/reconcile.service"; /** * One-shot reconciliation entrypoint (`npm run reconcile`): exits 0 when the ledger matches the chain at the - * sync checkpoint height, non-zero on any mismatch or misconfiguration, so it can gate a deploy. + * sync checkpoint height and the network aggregates match their recomputation from the leases table, + * non-zero on any mismatch or misconfiguration, so it can gate a deploy. */ async function main(): Promise { const logger = createOtelLogger({ context: "RECONCILE_CLI" }); @@ -24,8 +26,9 @@ async function main(): Promise { const sampleSize = parsed.data.RECONCILE_SAMPLE_SIZE; try { - const ok = await container.resolve(ReconcileService).reconcile(sampleSize === undefined ? {} : { sampleSize }); - process.exitCode = ok ? 0 : 1; + const bankOk = await container.resolve(ReconcileService).reconcile(sampleSize === undefined ? {} : { sampleSize }); + const networkOk = await container.resolve(NetworkStatsReconciler).reconcile(); + process.exitCode = bankOk && networkOk ? 0 : 1; } catch (error) { logger.error({ event: "RECONCILE_FATAL", error }); process.exitCode = 1; diff --git a/apps/chain-indexer/tsup.config.ts b/apps/chain-indexer/tsup.config.ts index 9457387734..13aab1bf0b 100644 --- a/apps/chain-indexer/tsup.config.ts +++ b/apps/chain-indexer/tsup.config.ts @@ -14,6 +14,7 @@ export default defineConfig(async overrideOptions => entry: { server: "./src/server.ts", reconcile: "./src/reconcile/reconcile.ts", + "recompute-usd": "./src/network/recompute-usd.ts", instrumentation: fileURLToPath(import.meta.resolve("@akashnetwork/instrumentation/register")) }, target: tsconfig.compilerOptions.target, From 1f618e7a8b80b5740fb200e1ea1307b937b0d4b3 Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Tue, 18 Aug 2026 11:56:25 +0530 Subject: [PATCH 13/14] fix(indexer): validate bme price attribute as dec string (#3605) A non-numeric price in a bme coin attribute previously flowed unchecked into the numeric(38,18) ledger columns, aborting the whole batch transaction instead of degrading to a parse warning like every other malformed attribute. Also documents that widening the scanned bme event-type set shifts replay ordinals. --- .../chain-indexer/src/bme/bme-deriver.spec.ts | 78 +++++++++++++++++++ apps/chain-indexer/src/bme/bme-deriver.ts | 12 ++- 2 files changed, 87 insertions(+), 3 deletions(-) diff --git a/apps/chain-indexer/src/bme/bme-deriver.spec.ts b/apps/chain-indexer/src/bme/bme-deriver.spec.ts index f6611ae2ff..5578c1d783 100644 --- a/apps/chain-indexer/src/bme/bme-deriver.spec.ts +++ b/apps/chain-indexer/src/bme/bme-deriver.spec.ts @@ -158,6 +158,30 @@ describe("deriveBmeChanges", () => { ]); }); + it("derives a mint status change from unquoted legacy attributes", () => { + const changes = deriveBmeChanges( + block({ + blockEvents: [ + event(STATUS_CHANGE_EVENT_TYPE, { + previous_status: "mint_status_healthy", + new_status: "mint_status_warning", + collateral_ratio: "1.75" + }) + ] + }) + ); + + expect(changes.warnings).toEqual([]); + expect(changes.changes).toEqual([ + expect.objectContaining({ + kind: "mintStatusChange", + previousStatus: "mint_status_healthy", + newStatus: "mint_status_warning", + collateralRatio: "1.75" + }) + ]); + }); + it("rejects a status change with a value outside the known mint statuses", () => { const changes = deriveBmeChanges( block({ @@ -208,6 +232,27 @@ describe("deriveBmeChanges", () => { ]); }); + it("drops a canceled record with a malformed coins_to_burn attribute", () => { + const changes = deriveBmeChanges( + block({ + blockEvents: [ + event(CANCELED_EVENT_TYPE, { + id: '{"denom":"uakt","to_denom":"uact","source":"bme","height":12000,"sequence":5}', + cancel_reason: '"insufficient_funds"', + owner: '"akash1owner"', + to: '"akash1dest"', + coins_to_burn: "not-json", + denom_to_mint: '"uact"' + }) + ] + }) + ); + + expect(changes.changes).toEqual([]); + expect(changes.warnings).toHaveLength(1); + expect(changes.warnings[0]).toContain(CANCELED_EVENT_TYPE); + }); + it("assigns ordinals across transaction events then block events in scan order", () => { const changes = deriveBmeChanges( block({ @@ -281,6 +326,39 @@ describe("deriveBmeChanges", () => { expect(changes.warnings).toHaveLength(1); }); + it("drops an executed event whose price is not a Dec string", () => { + const changes = deriveBmeChanges( + block({ + blockEvents: [ + event(EXECUTED_EVENT_TYPE, { + ...executedAttributes({ sequence: 1 }), + burned: '{"coin":{"denom":"uakt","amount":"1000000"},"price":"not-a-dec"}' + }) + ] + }) + ); + + expect(changes.changes).toEqual([]); + expect(changes.warnings).toHaveLength(1); + expect(changes.warnings[0]).toContain(EXECUTED_EVENT_TYPE); + }); + + it("treats a missing price as null on an otherwise valid coin", () => { + const changes = deriveBmeChanges( + block({ + blockEvents: [ + event(EXECUTED_EVENT_TYPE, { + ...executedAttributes({ sequence: 1 }), + burned: '{"coin":{"denom":"uakt","amount":"1000000"}}' + }) + ] + }) + ); + + expect(changes.warnings).toEqual([]); + expect(changes.changes).toEqual([expect.objectContaining({ burned: { denom: "uakt", amount: "1000000", price: null } })]); + }); + it("ignores vault funded and unrelated events", () => { const changes = deriveBmeChanges( block({ diff --git a/apps/chain-indexer/src/bme/bme-deriver.ts b/apps/chain-indexer/src/bme/bme-deriver.ts index c8213d2263..7e49cdd10e 100644 --- a/apps/chain-indexer/src/bme/bme-deriver.ts +++ b/apps/chain-indexer/src/bme/bme-deriver.ts @@ -66,8 +66,10 @@ type ParsedChange = { change: BmeChangeBody } | { error: string }; * mint status transitions and canceled records. Events of failed transactions are skipped; in practice * BME fires in the EndBlocker, but tx events are scanned too since the natural keys dedupe either way. * `ordinal` counts every BME-typed event in scan order — including ones that fail to parse — so a later - * parser fix replays with stable ordinals. Parse failures land in `warnings` for the writer to log; - * a malformed event must not halt the block. + * parser fix replays with stable ordinals. That stability only holds while the scanned event-type set is + * fixed: adding a BME event type shifts every later ordinal on replay, duplicating `bme_status_changes` + * rows unless previously derived rows are wiped first. Parse failures land in `warnings` for the writer + * to log; a malformed event must not halt the block. */ export function deriveBmeChanges(block: DecodedBlock): BmeBlockChanges { const changes: BmeChange[] = []; @@ -223,7 +225,11 @@ function parseCoinPrice(raw: string | undefined): BmeCoinPrice | null | undefine if (!coin) { return undefined; } - return { ...coin, price: asString(record?.price) }; + const price = asString(record?.price); + if (price !== null && !isDecString(price)) { + return undefined; + } + return { ...coin, price }; } function parseCoin(raw: string | undefined): BmeCoin | null | undefined { From a353d24f95011a5124520b4872d791170ccbc17e Mon Sep 17 00:00:00 2001 From: Maxime Beauchamp <15185355+baktun14@users.noreply.github.com> Date: Tue, 18 Aug 2026 18:56:31 +0530 Subject: [PATCH 14/14] fix(indexer): replay bme upgrade escrow denom migration exactly (#3606) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(indexer): replay bme upgrade escrow denom migration exactly The chain converted every open escrow in place at the BME upgrade (axlUSDC at par in the upgrade block, uakt at the oracle rate over the following EndBlocker drain) without per-account events, so rows open across the upgrade kept creation-era denoms and balances. The conversion is driven by the chain's observable record: the first native bme event marks the upgrade block, per-block burn/mint totals of the bme module account bind exactly how much each drain block converted, and the rate is the latest oracle price from strictly earlier blocks, matching the one-block lag of the chain's stored aggregate. Marker rows make every step exactly-once across replays and concurrent writers. Fixes CON-842 * fix(indexer): order act drain queue by decoded address bytes The upgrade seeds the uakt drain queue in the chain's pendingDenomMigrations order, which iterates raw AccAddress bytes. Sorting by the bech32 string does not reproduce that order — the bech32 charset is not ASCII-monotonic — so on a multi-owner chain the queue prefix consumed per drain block diverges from the chain's and trips the overshoot abort. Decode the address and sort by raw bytes. Also from review: reuse parseCoins for BME bank events so multi-coin amounts split across denoms, gate both lease conversion sums on the source balance to match the deployment branch, and collapse the three drained-marker claims into one helper. * refactor(indexer): derive act migration usdc denoms from mapping IBC_USDC_DENOMS re-listed the axlUSDC hashes DENOM_MAPPING already owns, so an axlUSDC denom change had two hand-synced edit points. Export DENOM_MAPPING and derive IBC_USDC_DENOMS from its uusdc entries. Also document the upgrade-block close ordering gap in the deliberate-gaps list: a deployment open going into the upgrade block but closed within it keeps its creation-era denom here, while the chain relabels it to uact at par in the BeginBlocker before the close runs (balances unaffected). * docs(indexer): record mainnet act migration heights and drain verification * refactor(indexer): use Buffer.compare for drain owner byte order The hand-rolled compareBytes reimplemented Buffer.compare's byte-then-length ordering, which accepts Uint8Array directly. Drop it and call Buffer.compare in the drain-sequence sort comparator. --- apps/chain-indexer/README.md | 8 + .../drizzle/0010_dark_violations.sql | 14 + .../drizzle/meta/0010_snapshot.json | 3139 +++++++++++++++++ apps/chain-indexer/drizzle/meta/_journal.json | 7 + apps/chain-indexer/src/akash/denom.ts | 2 +- .../src/bme/act-migration-convert.spec.ts | 76 + .../src/bme/act-migration-convert.ts | 75 + .../src/bme/act-migration-deriver.spec.ts | 143 + .../src/bme/act-migration-deriver.ts | 109 + .../src/bme/act-migration.service.spec.ts | 457 +++ .../src/bme/act-migration.service.ts | 602 ++++ apps/chain-indexer/src/bme/bme-deriver.ts | 2 +- apps/chain-indexer/src/db/schema.ts | 32 + .../pipeline/block-committer.service.spec.ts | 41 + .../src/pipeline/block-committer.service.ts | 24 +- .../pipeline/block-decoder.service.spec.ts | 21 +- .../src/pipeline/block-decoder.service.ts | 12 +- 17 files changed, 4753 insertions(+), 11 deletions(-) create mode 100644 apps/chain-indexer/drizzle/0010_dark_violations.sql create mode 100644 apps/chain-indexer/drizzle/meta/0010_snapshot.json create mode 100644 apps/chain-indexer/src/bme/act-migration-convert.spec.ts create mode 100644 apps/chain-indexer/src/bme/act-migration-convert.ts create mode 100644 apps/chain-indexer/src/bme/act-migration-deriver.spec.ts create mode 100644 apps/chain-indexer/src/bme/act-migration-deriver.ts create mode 100644 apps/chain-indexer/src/bme/act-migration.service.spec.ts create mode 100644 apps/chain-indexer/src/bme/act-migration.service.ts diff --git a/apps/chain-indexer/README.md b/apps/chain-indexer/README.md index 57e24f22c5..c9d1a45b8f 100644 --- a/apps/chain-indexer/README.md +++ b/apps/chain-indexer/README.md @@ -72,6 +72,14 @@ Every committed block also derives a balance ledger and an address activity log, The reason heuristic is deliberately MVP: coincident mint/burn/slash win first, then the module account on the holder's side of the movement (falling back to the counterparty's), then the denom. Per-deployment/lease attribution of escrow movements is left for later. +## ACT denom migration + +The BME network upgrade converted every open escrow in place — axlUSDC to uact at par in the upgrade block itself, uakt to uact at the oracle AKT/USD rate over the following blocks — without emitting per-account events. The pipeline replays that conversion from what the blocks do show (`src/bme/act-migration.service.ts`): the first native `akash.bme.v1.*` event marks the upgrade block, where axlUSDC deployments convert and every open uakt deployment is queued in the chain's drain order (owner address, then dseq). Each later block whose BME module account burned uakt and minted uact was a drain block: queue entries are consumed in order, converting deployments, their open leases, open/active bids and open/paused group resource prices, until the computed totals equal the block's burn/mint events exactly. The rate is the latest `EventPriceData` from strictly earlier blocks, matching the one-block lag of the oracle's stored aggregate. Overshooting the block's totals aborts the commit — a wrong rate cannot corrupt silently — and running out of queue first logs `ACT_MIGRATION_DRAIN_SHORTFALL`, which is expected on partial-window backfills and a red flag on a full sync. + +Both networks executed the migration in a single drain block, so the multi-block pacing is defensive rather than observed: v2.0.0 declares a 50-deployments-per-block cap but never increments its counter, and the whole queue drained at once. On mainnet the upgrade block is 26063777 (5,095.109965 axlUSDC converted at par) and the drain block is 26063781, which converted 9,663 deployments at the rate 0.584635140000000000 published at height 26063780. Replaying the queue reconstructed from historical chain state through this module's conversion math reproduces that block's burn of 44,862,222,630 uakt and mint of 26,228,026,358 uact to the exact uact, and only at that rate; the prices inside the drain block itself miss by tens of millions, which is the error the legacy indexer baked in. + +Every step claims an `indexer_state` marker (`act-migration:upgrade`, `act-migration:drain:`, `act-migration:drained`), so replays and overlapping writers skip already-applied steps. A module replay that rebuilds deployments from scratch must clear these markers and `act_migration_queue` along with the module's rows. Deliberate gaps, matching what the chain itself skipped or what this model does not track: deployments closed before their drain slot keep their creation-era denom (the chain skips them too), the chain's orphaned-escrow passes for closed deployments are not mirrored beyond open uusdc leases, a deployment open going into the upgrade block but closed by a transaction within that same block keeps its creation-era denom here (the chain converts it in the BeginBlocker before that transaction runs, so its final label is `uact` on chain — a par-rate relabel of an already-closed record, so balances are unaffected), and `deployments.deposit` scales by the rate as bookkeeping even though the chain keeps historical `transferred` entries in their original denom. + ## Reconciliation `npm run reconcile` proves the ledger matches the chain at the `sync` checkpoint height. It samples the highest-balance accounts, compares each against the node's bank balance at that height, and checks the ledger's per-denom totals against the chain's total supply; it exits non-zero on any mismatch or misconfiguration, so it can gate a deploy. Querying at the checkpoint rather than the moving tip keeps the comparison race-free, which requires an unpruned (archival) node — sandbox is archival. `RECONCILE_SAMPLE_SIZE` overrides the default sample of 100 accounts. diff --git a/apps/chain-indexer/drizzle/0010_dark_violations.sql b/apps/chain-indexer/drizzle/0010_dark_violations.sql new file mode 100644 index 0000000000..1117b845d3 --- /dev/null +++ b/apps/chain-indexer/drizzle/0010_dark_violations.sql @@ -0,0 +1,14 @@ +CREATE TABLE "akash"."act_migration_queue" ( + "position" integer PRIMARY KEY NOT NULL, + "deployment_id" bigint NOT NULL, + "converted_at_height" bigint +); +--> statement-breakpoint +CREATE TABLE "akash"."act_migration_state" ( + "id" integer PRIMARY KEY NOT NULL, + "last_akt_usd_price" numeric(38, 18), + "last_price_height" bigint, + CONSTRAINT "act_migration_state_singleton_check" CHECK ("akash"."act_migration_state"."id" = 1) +); +--> statement-breakpoint +ALTER TABLE "akash"."act_migration_queue" ADD CONSTRAINT "act_migration_queue_deployment_id_deployments_id_fk" FOREIGN KEY ("deployment_id") REFERENCES "akash"."deployments"("id") ON DELETE no action ON UPDATE no action; \ No newline at end of file diff --git a/apps/chain-indexer/drizzle/meta/0010_snapshot.json b/apps/chain-indexer/drizzle/meta/0010_snapshot.json new file mode 100644 index 0000000000..23ecf881a0 --- /dev/null +++ b/apps/chain-indexer/drizzle/meta/0010_snapshot.json @@ -0,0 +1,3139 @@ +{ + "id": "7f384fb1-b087-4028-8a96-497ec7fa1f97", + "prevId": "080d0308-11dd-4220-855e-30746d3813e4", + "version": "7", + "dialect": "postgresql", + "tables": { + "cosmos.account_balances": { + "name": "account_balances", + "schema": "cosmos", + "columns": { + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "amount": { + "name": "amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "account_balances_account_id_accounts_id_fk": { + "name": "account_balances_account_id_accounts_id_fk", + "tableFrom": "account_balances", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_balances_account_id_denom_pk": { + "name": "account_balances_account_id_denom_pk", + "columns": [ + "account_id", + "denom" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.account_txs": { + "name": "account_txs", + "schema": "cosmos", + "columns": { + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "account_tx_role", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "account_txs_account_id_accounts_id_fk": { + "name": "account_txs_account_id_accounts_id_fk", + "tableFrom": "account_txs", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_txs_account_id_height_tx_index_role_pk": { + "name": "account_txs_account_id_height_tx_index_role_pk", + "columns": [ + "account_id", + "height", + "tx_index", + "role" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.accounts": { + "name": "accounts", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_number": { + "name": "account_number", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "account_type": { + "name": "account_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_module_account": { + "name": "is_module_account", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": { + "accounts_address_idx": { + "name": "accounts_address_idx", + "columns": [ + { + "expression": "address", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.act_migration_queue": { + "name": "act_migration_queue", + "schema": "akash", + "columns": { + "position": { + "name": "position", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "deployment_id": { + "name": "deployment_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "converted_at_height": { + "name": "converted_at_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "act_migration_queue_deployment_id_deployments_id_fk": { + "name": "act_migration_queue_deployment_id_deployments_id_fk", + "tableFrom": "act_migration_queue", + "tableTo": "deployments", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.act_migration_state": { + "name": "act_migration_state", + "schema": "akash", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "last_akt_usd_price": { + "name": "last_akt_usd_price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": false + }, + "last_price_height": { + "name": "last_price_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "act_migration_state_singleton_check": { + "name": "act_migration_state_singleton_check", + "value": "\"akash\".\"act_migration_state\".\"id\" = 1" + } + }, + "isRLSEnabled": false + }, + "cosmos.balance_changes": { + "name": "balance_changes", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delta": { + "name": "delta", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "balance_after": { + "name": "balance_after", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "balance_change_reason", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "event_index": { + "name": "event_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "counterparty_account_id": { + "name": "counterparty_account_id", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "balance_changes_account_denom_height_idx": { + "name": "balance_changes_account_denom_height_idx", + "columns": [ + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "denom", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "balance_changes_height_event_index_idx": { + "name": "balance_changes_height_event_index_idx", + "columns": [ + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_index", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "balance_changes_account_id_accounts_id_fk": { + "name": "balance_changes_account_id_accounts_id_fk", + "tableFrom": "balance_changes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "balance_changes_counterparty_account_id_accounts_id_fk": { + "name": "balance_changes_counterparty_account_id_accounts_id_fk", + "tableFrom": "balance_changes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "counterparty_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.bids": { + "name": "bids", + "schema": "akash", + "columns": { + "deployment_id": { + "name": "deployment_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gseq": { + "name": "gseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "oseq": { + "name": "oseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "bseq": { + "name": "bseq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "provider_account_id": { + "name": "provider_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "price": { + "name": "price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "bid_state", + "typeSchema": "akash", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "created_height": { + "name": "created_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "closed_height": { + "name": "closed_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "bids_deployment_id_deployments_id_fk": { + "name": "bids_deployment_id_deployments_id_fk", + "tableFrom": "bids", + "tableTo": "deployments", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "bids_provider_account_id_accounts_id_fk": { + "name": "bids_provider_account_id_accounts_id_fk", + "tableFrom": "bids", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "provider_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "bids_deployment_id_gseq_oseq_bseq_provider_account_id_pk": { + "name": "bids_deployment_id_gseq_oseq_bseq_provider_account_id_pk", + "columns": [ + "deployment_id", + "gseq", + "oseq", + "bseq", + "provider_account_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.blocks": { + "name": "blocks", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "datetime": { + "name": "datetime", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "parent_hash": { + "name": "parent_hash", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "proposer_address": { + "name": "proposer_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tx_count": { + "name": "tx_count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.bme_canceled_records": { + "name": "bme_canceled_records", + "schema": "akash", + "columns": { + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "to_denom": { + "name": "to_denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "record_height": { + "name": "record_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "sequence": { + "name": "sequence", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cancel_reason": { + "name": "cancel_reason", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "owner_account_id": { + "name": "owner_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "to_account_id": { + "name": "to_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "coins_to_burn_denom": { + "name": "coins_to_burn_denom", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "coins_to_burn_amount": { + "name": "coins_to_burn_amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "denom_to_mint": { + "name": "denom_to_mint", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "bme_canceled_records_height_idx": { + "name": "bme_canceled_records_height_idx", + "columns": [ + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "bme_canceled_records_owner_account_id_accounts_id_fk": { + "name": "bme_canceled_records_owner_account_id_accounts_id_fk", + "tableFrom": "bme_canceled_records", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "owner_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "bme_canceled_records_to_account_id_accounts_id_fk": { + "name": "bme_canceled_records_to_account_id_accounts_id_fk", + "tableFrom": "bme_canceled_records", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "to_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "bme_canceled_records_record_height_sequence_denom_to_denom_source_pk": { + "name": "bme_canceled_records_record_height_sequence_denom_to_denom_source_pk", + "columns": [ + "record_height", + "sequence", + "denom", + "to_denom", + "source" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.bme_ledger_records": { + "name": "bme_ledger_records", + "schema": "akash", + "columns": { + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "to_denom": { + "name": "to_denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "record_height": { + "name": "record_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "sequence": { + "name": "sequence", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "burned_from_account_id": { + "name": "burned_from_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "minted_to_account_id": { + "name": "minted_to_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "burned_denom": { + "name": "burned_denom", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "burned_amount": { + "name": "burned_amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "burned_price": { + "name": "burned_price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": false + }, + "minted_denom": { + "name": "minted_denom", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "minted_amount": { + "name": "minted_amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "minted_price": { + "name": "minted_price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": false + }, + "spread_denom": { + "name": "spread_denom", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "spread_amount": { + "name": "spread_amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "remint_credit_issued_amount": { + "name": "remint_credit_issued_amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "remint_credit_accrued_amount": { + "name": "remint_credit_accrued_amount", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "bme_ledger_records_height_idx": { + "name": "bme_ledger_records_height_idx", + "columns": [ + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "bme_ledger_records_burned_denom_height_idx": { + "name": "bme_ledger_records_burned_denom_height_idx", + "columns": [ + { + "expression": "burned_denom", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "bme_ledger_records_minted_denom_height_idx": { + "name": "bme_ledger_records_minted_denom_height_idx", + "columns": [ + { + "expression": "minted_denom", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "bme_ledger_records_burned_from_account_id_accounts_id_fk": { + "name": "bme_ledger_records_burned_from_account_id_accounts_id_fk", + "tableFrom": "bme_ledger_records", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "burned_from_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "bme_ledger_records_minted_to_account_id_accounts_id_fk": { + "name": "bme_ledger_records_minted_to_account_id_accounts_id_fk", + "tableFrom": "bme_ledger_records", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "minted_to_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "bme_ledger_records_record_height_sequence_denom_to_denom_source_pk": { + "name": "bme_ledger_records_record_height_sequence_denom_to_denom_source_pk", + "columns": [ + "record_height", + "sequence", + "denom", + "to_denom", + "source" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.bme_status_changes": { + "name": "bme_status_changes", + "schema": "akash", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ordinal": { + "name": "ordinal", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "previous_status": { + "name": "previous_status", + "type": "bme_mint_status", + "typeSchema": "akash", + "primaryKey": false, + "notNull": true + }, + "new_status": { + "name": "new_status", + "type": "bme_mint_status", + "typeSchema": "akash", + "primaryKey": false, + "notNull": true + }, + "collateral_ratio": { + "name": "collateral_ratio", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "bme_status_changes_height_ordinal_pk": { + "name": "bme_status_changes_height_ordinal_pk", + "columns": [ + "height", + "ordinal" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.daily_prices": { + "name": "daily_prices", + "schema": "akash", + "columns": { + "date": { + "name": "date", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "price": { + "name": "price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "daily_prices_date_denom_pk": { + "name": "daily_prices_date_denom_pk", + "columns": [ + "date", + "denom" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.delegations": { + "name": "delegations", + "schema": "cosmos", + "columns": { + "delegator_account_id": { + "name": "delegator_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "validator_operator_address": { + "name": "validator_operator_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "shares": { + "name": "shares", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "delegations_delegator_account_id_accounts_id_fk": { + "name": "delegations_delegator_account_id_accounts_id_fk", + "tableFrom": "delegations", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "delegator_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "delegations_delegator_account_id_validator_operator_address_pk": { + "name": "delegations_delegator_account_id_validator_operator_address_pk", + "columns": [ + "delegator_account_id", + "validator_operator_address" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.deployment_events": { + "name": "deployment_events", + "schema": "akash", + "columns": { + "deployment_id": { + "name": "deployment_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ordinal": { + "name": "ordinal", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "msg_index": { + "name": "msg_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "deployment_event_type", + "typeSchema": "akash", + "primaryKey": false, + "notNull": true + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "deployment_events_deployment_id_deployments_id_fk": { + "name": "deployment_events_deployment_id_deployments_id_fk", + "tableFrom": "deployment_events", + "tableTo": "deployments", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "deployment_events_deployment_id_height_ordinal_pk": { + "name": "deployment_events_deployment_id_height_ordinal_pk", + "columns": [ + "deployment_id", + "height", + "ordinal" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.deployment_group_resources": { + "name": "deployment_group_resources", + "schema": "akash", + "columns": { + "deployment_group_id": { + "name": "deployment_group_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "idx": { + "name": "idx", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "cpu_units": { + "name": "cpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gpu_units": { + "name": "gpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gpu_vendor": { + "name": "gpu_vendor", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "gpu_model": { + "name": "gpu_model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "memory_bytes": { + "name": "memory_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ephemeral_storage_bytes": { + "name": "ephemeral_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "persistent_storage_bytes": { + "name": "persistent_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "price": { + "name": "price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "price_denom": { + "name": "price_denom", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "deployment_group_resources_deployment_group_id_deployment_groups_id_fk": { + "name": "deployment_group_resources_deployment_group_id_deployment_groups_id_fk", + "tableFrom": "deployment_group_resources", + "tableTo": "deployment_groups", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_group_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "deployment_group_resources_deployment_group_id_idx_pk": { + "name": "deployment_group_resources_deployment_group_id_idx_pk", + "columns": [ + "deployment_group_id", + "idx" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.deployment_groups": { + "name": "deployment_groups", + "schema": "akash", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "deployment_id": { + "name": "deployment_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gseq": { + "name": "gseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "group_state", + "typeSchema": "akash", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "closed_height": { + "name": "closed_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "deployment_groups_deployment_gseq_idx": { + "name": "deployment_groups_deployment_gseq_idx", + "columns": [ + { + "expression": "deployment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "gseq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "deployment_groups_deployment_id_deployments_id_fk": { + "name": "deployment_groups_deployment_id_deployments_id_fk", + "tableFrom": "deployment_groups", + "tableTo": "deployments", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.deployments": { + "name": "deployments", + "schema": "akash", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "owner_account_id": { + "name": "owner_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "dseq": { + "name": "dseq", + "type": "numeric(20, 0)", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deposit": { + "name": "deposit", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "balance": { + "name": "balance", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "withdrawn_amount": { + "name": "withdrawn_amount", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "block_rate": { + "name": "block_rate", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "last_withdraw_height": { + "name": "last_withdraw_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "last_processed_height": { + "name": "last_processed_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_height": { + "name": "created_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "closed_height": { + "name": "closed_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "closed_at": { + "name": "closed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "close_reason": { + "name": "close_reason", + "type": "deployment_close_reason", + "typeSchema": "akash", + "primaryKey": false, + "notNull": false + }, + "cpu_units": { + "name": "cpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gpu_units": { + "name": "gpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "memory_bytes": { + "name": "memory_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ephemeral_storage_bytes": { + "name": "ephemeral_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "persistent_storage_bytes": { + "name": "persistent_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "deployments_owner_dseq_idx": { + "name": "deployments_owner_dseq_idx", + "columns": [ + { + "expression": "owner_account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dseq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "deployments_owner_created_idx": { + "name": "deployments_owner_created_idx", + "columns": [ + { + "expression": "owner_account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "deployments_open_idx": { + "name": "deployments_open_idx", + "columns": [ + { + "expression": "created_height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"akash\".\"deployments\".\"closed_height\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "deployments_owner_account_id_accounts_id_fk": { + "name": "deployments_owner_account_id_accounts_id_fk", + "tableFrom": "deployments", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "owner_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.indexer_state": { + "name": "indexer_state", + "schema": "", + "columns": { + "stream": { + "name": "stream", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "last_height": { + "name": "last_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.leases": { + "name": "leases", + "schema": "akash", + "columns": { + "deployment_id": { + "name": "deployment_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "deployment_group_id": { + "name": "deployment_group_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gseq": { + "name": "gseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "oseq": { + "name": "oseq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "bseq": { + "name": "bseq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "provider_account_id": { + "name": "provider_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "price": { + "name": "price", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "denom": { + "name": "denom", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "balance": { + "name": "balance", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "withdrawn_amount": { + "name": "withdrawn_amount", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "predicted_closed_height": { + "name": "predicted_closed_height", + "type": "numeric(30, 0)", + "primaryKey": false, + "notNull": true + }, + "created_height": { + "name": "created_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "closed_height": { + "name": "closed_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "closed_at": { + "name": "closed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "cpu_units": { + "name": "cpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gpu_units": { + "name": "gpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "memory_bytes": { + "name": "memory_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ephemeral_storage_bytes": { + "name": "ephemeral_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "persistent_storage_bytes": { + "name": "persistent_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "leases_provider_idx": { + "name": "leases_provider_idx", + "columns": [ + { + "expression": "provider_account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "closed_height", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "leases_open_idx": { + "name": "leases_open_idx", + "columns": [ + { + "expression": "deployment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"akash\".\"leases\".\"closed_height\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "leases_deployment_id_deployments_id_fk": { + "name": "leases_deployment_id_deployments_id_fk", + "tableFrom": "leases", + "tableTo": "deployments", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "leases_deployment_group_id_deployment_groups_id_fk": { + "name": "leases_deployment_group_id_deployment_groups_id_fk", + "tableFrom": "leases", + "tableTo": "deployment_groups", + "schemaTo": "akash", + "columnsFrom": [ + "deployment_group_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "leases_provider_account_id_accounts_id_fk": { + "name": "leases_provider_account_id_accounts_id_fk", + "tableFrom": "leases", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "provider_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "leases_deployment_id_gseq_oseq_bseq_provider_account_id_pk": { + "name": "leases_deployment_id_gseq_oseq_bseq_provider_account_id_pk", + "columns": [ + "deployment_id", + "gseq", + "oseq", + "bseq", + "provider_account_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.message_dead_letters": { + "name": "message_dead_letters", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type_id": { + "name": "type_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "raw": { + "name": "raw", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "message_dead_letters_type_id_idx": { + "name": "message_dead_letters_type_id_idx", + "columns": [ + { + "expression": "type_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "message_dead_letters_type_id_message_types_id_fk": { + "name": "message_dead_letters_type_id_message_types_id_fk", + "tableFrom": "message_dead_letters", + "tableTo": "message_types", + "schemaTo": "cosmos", + "columnsFrom": [ + "type_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "message_dead_letters_height_tx_index_index_pk": { + "name": "message_dead_letters_height_tx_index_index_pk", + "columns": [ + "height", + "tx_index", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.message_types": { + "name": "message_types", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "message_types_type_idx": { + "name": "message_types_type_idx", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.messages": { + "name": "messages", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type_id": { + "name": "type_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "body": { + "name": "body", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "messages_type_id_idx": { + "name": "messages_type_id_idx", + "columns": [ + { + "expression": "type_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "messages_type_id_message_types_id_fk": { + "name": "messages_type_id_message_types_id_fk", + "tableFrom": "messages", + "tableTo": "message_types", + "schemaTo": "cosmos", + "columnsFrom": [ + "type_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messages_height_tx_index_index_pk": { + "name": "messages_height_tx_index_index_pk", + "columns": [ + "height", + "tx_index", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.network_rollups": { + "name": "network_rollups", + "schema": "akash", + "columns": { + "date": { + "name": "date", + "type": "date", + "primaryKey": true, + "notNull": true + }, + "close_height": { + "name": "close_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "close_at": { + "name": "close_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "active_lease_count": { + "name": "active_lease_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "total_lease_count": { + "name": "total_lease_count", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "daily_lease_count": { + "name": "daily_lease_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "active_provider_count": { + "name": "active_provider_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "active_cpu_units": { + "name": "active_cpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "active_gpu_units": { + "name": "active_gpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "active_memory_bytes": { + "name": "active_memory_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "active_ephemeral_storage_bytes": { + "name": "active_ephemeral_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "active_persistent_storage_bytes": { + "name": "active_persistent_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "total_uakt_spent": { + "name": "total_uakt_spent", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "total_uusdc_spent": { + "name": "total_uusdc_spent", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "total_uact_spent": { + "name": "total_uact_spent", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "daily_uakt_spent": { + "name": "daily_uakt_spent", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "daily_uusdc_spent": { + "name": "daily_uusdc_spent", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "daily_uact_spent": { + "name": "daily_uact_spent", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true + }, + "daily_usd_spent": { + "name": "daily_usd_spent", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": false + }, + "akt_price_used": { + "name": "akt_price_used", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": false + }, + "usd_computed_at": { + "name": "usd_computed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.network_state": { + "name": "network_state", + "schema": "akash", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "last_aggregated_height": { + "name": "last_aggregated_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "last_aggregated_at": { + "name": "last_aggregated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "active_lease_count": { + "name": "active_lease_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_lease_count": { + "name": "total_lease_count", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "active_provider_count": { + "name": "active_provider_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "active_cpu_units": { + "name": "active_cpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "active_gpu_units": { + "name": "active_gpu_units", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "active_memory_bytes": { + "name": "active_memory_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "active_ephemeral_storage_bytes": { + "name": "active_ephemeral_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "active_persistent_storage_bytes": { + "name": "active_persistent_storage_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_uakt_spent": { + "name": "total_uakt_spent", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "total_uusdc_spent": { + "name": "total_uusdc_spent", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "total_uact_spent": { + "name": "total_uact_spent", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "network_state_singleton_check": { + "name": "network_state_singleton_check", + "value": "\"akash\".\"network_state\".\"id\" = 1" + } + }, + "isRLSEnabled": false + }, + "cosmos.proposal_deposits": { + "name": "proposal_deposits", + "schema": "cosmos", + "columns": { + "proposal_id": { + "name": "proposal_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "depositor_account_id": { + "name": "depositor_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "amount": { + "name": "amount", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "proposal_deposits_depositor_account_id_accounts_id_fk": { + "name": "proposal_deposits_depositor_account_id_accounts_id_fk", + "tableFrom": "proposal_deposits", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "depositor_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "proposal_deposits_proposal_id_depositor_account_id_height_pk": { + "name": "proposal_deposits_proposal_id_depositor_account_id_height_pk", + "columns": [ + "proposal_id", + "depositor_account_id", + "height" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.proposal_votes": { + "name": "proposal_votes", + "schema": "cosmos", + "columns": { + "proposal_id": { + "name": "proposal_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "voter_account_id": { + "name": "voter_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "options": { + "name": "options", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "proposal_votes_voter_account_id_accounts_id_fk": { + "name": "proposal_votes_voter_account_id_accounts_id_fk", + "tableFrom": "proposal_votes", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "voter_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "proposal_votes_proposal_id_voter_account_id_pk": { + "name": "proposal_votes_proposal_id_voter_account_id_pk", + "columns": [ + "proposal_id", + "voter_account_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.proposals": { + "name": "proposals", + "schema": "cosmos", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "proposer_account_id": { + "name": "proposer_account_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "messages": { + "name": "messages", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "proposal_status", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": true + }, + "submit_time": { + "name": "submit_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deposit_end_time": { + "name": "deposit_end_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "voting_start_time": { + "name": "voting_start_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "voting_end_time": { + "name": "voting_end_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "total_deposit": { + "name": "total_deposit", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "final_tally_yes": { + "name": "final_tally_yes", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "final_tally_abstain": { + "name": "final_tally_abstain", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "final_tally_no": { + "name": "final_tally_no", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "final_tally_no_with_veto": { + "name": "final_tally_no_with_veto", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "submit_height": { + "name": "submit_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "proposals_proposer_account_id_accounts_id_fk": { + "name": "proposals_proposer_account_id_accounts_id_fk", + "tableFrom": "proposals", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "proposer_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.provider_audit_signatures": { + "name": "provider_audit_signatures", + "schema": "akash", + "columns": { + "owner_account_id": { + "name": "owner_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "auditor_account_id": { + "name": "auditor_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "provider_audit_signatures_owner_account_id_accounts_id_fk": { + "name": "provider_audit_signatures_owner_account_id_accounts_id_fk", + "tableFrom": "provider_audit_signatures", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "owner_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "provider_audit_signatures_auditor_account_id_accounts_id_fk": { + "name": "provider_audit_signatures_auditor_account_id_accounts_id_fk", + "tableFrom": "provider_audit_signatures", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "auditor_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "provider_audit_signatures_owner_account_id_auditor_account_id_key_pk": { + "name": "provider_audit_signatures_owner_account_id_auditor_account_id_key_pk", + "columns": [ + "owner_account_id", + "auditor_account_id", + "key" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "akash.providers": { + "name": "providers", + "schema": "akash", + "columns": { + "owner_account_id": { + "name": "owner_account_id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "host_uri": { + "name": "host_uri", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attributes": { + "name": "attributes", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "last_processed_height": { + "name": "last_processed_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_height": { + "name": "created_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "updated_height": { + "name": "updated_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "deleted_height": { + "name": "deleted_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "providers_owner_account_id_accounts_id_fk": { + "name": "providers_owner_account_id_accounts_id_fk", + "tableFrom": "providers", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "owner_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.transactions": { + "name": "transactions", + "schema": "cosmos", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "index": { + "name": "index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "code": { + "name": "code", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "gas_used": { + "name": "gas_used", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "gas_wanted": { + "name": "gas_wanted", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "fee": { + "name": "fee", + "type": "jsonb", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "transactions_hash_idx": { + "name": "transactions_hash_idx", + "columns": [ + { + "expression": "hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "transactions_height_index_pk": { + "name": "transactions_height_index_pk", + "columns": [ + "height", + "index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.unbonding_delegations": { + "name": "unbonding_delegations", + "schema": "cosmos", + "columns": { + "delegator_account_id": { + "name": "delegator_account_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "validator_operator_address": { + "name": "validator_operator_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "creation_height": { + "name": "creation_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "completion_time": { + "name": "completion_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "initial_balance": { + "name": "initial_balance", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + }, + "balance": { + "name": "balance", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "unbonding_delegations_delegator_account_id_accounts_id_fk": { + "name": "unbonding_delegations_delegator_account_id_accounts_id_fk", + "tableFrom": "unbonding_delegations", + "tableTo": "accounts", + "schemaTo": "cosmos", + "columnsFrom": [ + "delegator_account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "unbonding_delegations_delegator_account_id_validator_operator_address_creation_height_pk": { + "name": "unbonding_delegations_delegator_account_id_validator_operator_address_creation_height_pk", + "columns": [ + "delegator_account_id", + "validator_operator_address", + "creation_height" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "cosmos.validators": { + "name": "validators", + "schema": "cosmos", + "columns": { + "operator_address": { + "name": "operator_address", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_address": { + "name": "account_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "hex_address": { + "name": "hex_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "moniker": { + "name": "moniker", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "identity": { + "name": "identity", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "details": { + "name": "details", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "security_contact": { + "name": "security_contact", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commission_rate": { + "name": "commission_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "commission_max_rate": { + "name": "commission_max_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "commission_max_change_rate": { + "name": "commission_max_change_rate", + "type": "numeric(20, 18)", + "primaryKey": false, + "notNull": false + }, + "min_self_delegation": { + "name": "min_self_delegation", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "jailed": { + "name": "jailed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "status": { + "name": "status", + "type": "validator_status", + "typeSchema": "cosmos", + "primaryKey": false, + "notNull": false + }, + "tokens": { + "name": "tokens", + "type": "numeric(38, 0)", + "primaryKey": false, + "notNull": false + }, + "delegator_shares": { + "name": "delegator_shares", + "type": "numeric(38, 18)", + "primaryKey": false, + "notNull": false + }, + "unbonding_height": { + "name": "unbonding_height", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "unbonding_time": { + "name": "unbonding_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "cosmos.account_tx_role": { + "name": "account_tx_role", + "schema": "cosmos", + "values": [ + "signer", + "sender", + "receiver" + ] + }, + "cosmos.balance_change_reason": { + "name": "balance_change_reason", + "schema": "cosmos", + "values": [ + "genesis", + "transfer", + "fee", + "reward", + "commission", + "slash", + "gov", + "ibc", + "escrow", + "bme", + "mint", + "burn", + "staking" + ] + }, + "akash.bid_state": { + "name": "bid_state", + "schema": "akash", + "values": [ + "open", + "active", + "closed" + ] + }, + "akash.bme_mint_status": { + "name": "bme_mint_status", + "schema": "akash", + "values": [ + "mint_status_unspecified", + "mint_status_healthy", + "mint_status_warning", + "mint_status_halt_cr", + "mint_status_halt_oracle" + ] + }, + "akash.deployment_close_reason": { + "name": "deployment_close_reason", + "schema": "akash", + "values": [ + "close_message", + "overdrawn", + "close_event" + ] + }, + "akash.deployment_event_type": { + "name": "deployment_event_type", + "schema": "akash", + "values": [ + "created", + "deposited", + "updated", + "closed", + "group_closed", + "group_paused", + "group_started", + "bid_created", + "bid_closed", + "lease_created", + "lease_closed", + "lease_withdrawn" + ] + }, + "akash.group_state": { + "name": "group_state", + "schema": "akash", + "values": [ + "open", + "paused", + "closed" + ] + }, + "cosmos.proposal_status": { + "name": "proposal_status", + "schema": "cosmos", + "values": [ + "deposit_period", + "voting_period", + "passed", + "rejected", + "failed" + ] + }, + "cosmos.validator_status": { + "name": "validator_status", + "schema": "cosmos", + "values": [ + "unbonded", + "unbonding", + "bonded" + ] + }, + "cosmos.vote_option": { + "name": "vote_option", + "schema": "cosmos", + "values": [ + "yes", + "abstain", + "no", + "no_with_veto" + ] + } + }, + "schemas": { + "akash": "akash", + "cosmos": "cosmos" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/chain-indexer/drizzle/meta/_journal.json b/apps/chain-indexer/drizzle/meta/_journal.json index 669c3151c7..a61367b318 100644 --- a/apps/chain-indexer/drizzle/meta/_journal.json +++ b/apps/chain-indexer/drizzle/meta/_journal.json @@ -71,6 +71,13 @@ "when": 1786993129072, "tag": "0009_loud_blue_marvel", "breakpoints": true + }, + { + "idx": 10, + "version": "7", + "when": 1787037861240, + "tag": "0010_dark_violations", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/chain-indexer/src/akash/denom.ts b/apps/chain-indexer/src/akash/denom.ts index 84227d9441..f4c0dd18a1 100644 --- a/apps/chain-indexer/src/akash/denom.ts +++ b/apps/chain-indexer/src/akash/denom.ts @@ -1,5 +1,5 @@ /** The IBC denoms deployments are funded with, mapped to their base denom (mirrors the legacy indexer's mapping). */ -const DENOM_MAPPING = new Map([ +export const DENOM_MAPPING = new Map([ ["uakt", "uakt"], ["uact", "uact"], ["ibc/028CD1864059EEFB48A6048376165318E3E82C234390AE5A6D7B22001725B06E", "uusdc"], diff --git a/apps/chain-indexer/src/bme/act-migration-convert.spec.ts b/apps/chain-indexer/src/bme/act-migration-convert.spec.ts new file mode 100644 index 0000000000..b2773e50de --- /dev/null +++ b/apps/chain-indexer/src/bme/act-migration-convert.spec.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from "vitest"; + +import { + conversionBankContribution, + convertDeploymentAmounts, + convertLeaseAmounts, + convertPriceAmount, + parseRate, + RATE_ONE +} from "@src/bme/act-migration-convert"; + +/** The rate the sandbox drain actually used at block 2552660: the price published at 2552658. */ +const SANDBOX_RATE = parseRate("0.626310480000000000"); + +describe("act-migration-convert", () => { + describe("parseRate", () => { + it("rejects zero and negative rates", () => { + expect(() => parseRate("0")).toThrow("Invalid AKT/USD rate"); + expect(() => parseRate("-1.5")).toThrow("Invalid AKT/USD rate"); + }); + }); + + describe("convertPriceAmount", () => { + it("scales a Dec price by the rate, reproducing the on-chain group price conversion", () => { + expect(convertPriceAmount("1000", SANDBOX_RATE)).toBe("626.31048"); + }); + + it("is exact at rate one, so the axlUSDC pass changes nothing", () => { + expect(convertPriceAmount("626.310480000000000001", RATE_ONE)).toBe("626.310480000000000001"); + }); + }); + + describe("convertDeploymentAmounts", () => { + it("scales Dec columns and truncates the integer deposit like the chain's Coin conversion", () => { + const converted = convertDeploymentAmounts({ balance: "5000000.5", deposit: "5000000", withdrawnAmount: "1000000", blockRate: "1000" }, SANDBOX_RATE); + + expect(converted).toEqual({ + balance: "3131552.71315524", + deposit: "3131552", + withdrawnAmount: "626310.48", + blockRate: "626.31048" + }); + }); + + it("converts negative overdrawn balances too, matching the chain", () => { + const converted = convertDeploymentAmounts({ balance: "-100", deposit: "0", withdrawnAmount: "0", blockRate: "0" }, SANDBOX_RATE); + + expect(converted.balance).toBe("-62.631048"); + }); + }); + + describe("convertLeaseAmounts", () => { + it("scales price and balance as Decs and the withdrawn amount as a truncated integer Coin", () => { + const converted = convertLeaseAmounts({ price: "1000", balance: "500.5", withdrawnAmount: "999999" }, SANDBOX_RATE); + + expect(converted.price).toBe("626.31048"); + expect(converted.balance).toBe("313.46839524"); + expect(converted.withdrawnAmount).toBe("626309"); + }); + }); + + describe("conversionBankContribution", () => { + it("truncates the deployment's summed funds and payment balances once, like the chain's per-deployment TruncateDecimal", () => { + const contribution = conversionBankContribution({ balance: "100.7", leaseBalances: ["10.5"] }, SANDBOX_RATE); + + expect(contribution.burned).toBe(111n); + expect(contribution.minted).toBe(69n); + }); + + it("excludes negative balances from the burn and mint totals but keeps non-negative ones", () => { + const contribution = conversionBankContribution({ balance: "-5", leaseBalances: ["10", "-3"] }, RATE_ONE); + + expect(contribution).toEqual({ burned: 10n, minted: 10n }); + }); + }); +}); diff --git a/apps/chain-indexer/src/bme/act-migration-convert.ts b/apps/chain-indexer/src/bme/act-migration-convert.ts new file mode 100644 index 0000000000..205c3f3b90 --- /dev/null +++ b/apps/chain-indexer/src/bme/act-migration-convert.ts @@ -0,0 +1,75 @@ +import { decFromString, decMul, decToString, decTruncateInt } from "@src/akash/dec"; + +/** + * Ports the per-deployment math of the chain's denom migration (`x/deployment/migrate/v7`): every + * Dec amount multiplies by the AKT/USD rate with LegacyDec rounding, and integer Coin amounts use + * the chain's `rate.MulInt().TruncateInt()`. The axlUSDC pass is the same code path at rate 1. + */ +export const RATE_ONE = decFromString("1"); + +export function parseRate(rate: string): bigint { + const atomics = decFromString(rate); + if (atomics <= 0n) { + throw new Error(`Invalid AKT/USD rate for ACT migration: ${rate}`); + } + return atomics; +} + +export function convertDeploymentAmounts( + input: { balance: string; deposit: string; withdrawnAmount: string; blockRate: string }, + rate: bigint +): { balance: string; deposit: string; withdrawnAmount: string; blockRate: string } { + return { + balance: decToString(decMul(decFromString(input.balance), rate)), + deposit: convertIntCoinAmount(input.deposit, rate), + withdrawnAmount: decToString(decMul(decFromString(input.withdrawnAmount), rate)), + blockRate: decToString(decMul(decFromString(input.blockRate), rate)) + }; +} + +/** `withdrawn` mirrors the on-chain integer Coin: convert its integral value, truncating like the chain. */ +export function convertLeaseAmounts( + input: { price: string; balance: string; withdrawnAmount: string }, + rate: bigint +): { price: string; balance: string; withdrawnAmount: string } { + return { + price: decToString(decMul(decFromString(input.price), rate)), + balance: decToString(decMul(decFromString(input.balance), rate)), + withdrawnAmount: convertIntCoinAmount(decTruncateInt(decFromString(input.withdrawnAmount)).toString(), rate) + }; +} + +export function convertPriceAmount(price: string, rate: bigint): string { + return decToString(decMul(decFromString(price), rate)); +} + +/** + * What this deployment contributes to the block's burn/mint bank events: the chain sums the + * non-negative escrow funds and payment balances as Decs, converts them, and truncates each + * deployment's totals to whole coins before adding them to the per-block aggregate. + */ +export function conversionBankContribution(input: { balance: string; leaseBalances: string[] }, rate: bigint): { burned: bigint; minted: bigint } { + let sourceSum = 0n; + let convertedSum = 0n; + + const funds = decFromString(input.balance); + if (funds >= 0n) { + sourceSum += funds; + convertedSum += decMul(funds, rate); + } + + for (const leaseBalance of input.leaseBalances) { + const balance = decFromString(leaseBalance); + if (balance >= 0n) { + sourceSum += balance; + convertedSum += decMul(balance, rate); + } + } + + return { burned: decTruncateInt(sourceSum), minted: decTruncateInt(convertedSum) }; +} + +/** The chain's `convertCoin`: `rate.MulInt(amount).TruncateInt()` over an integer micro-denom amount. */ +function convertIntCoinAmount(amount: string, rate: bigint): string { + return decTruncateInt(rate * BigInt(amount)).toString(); +} diff --git a/apps/chain-indexer/src/bme/act-migration-deriver.spec.ts b/apps/chain-indexer/src/bme/act-migration-deriver.spec.ts new file mode 100644 index 0000000000..8c9457e027 --- /dev/null +++ b/apps/chain-indexer/src/bme/act-migration-deriver.spec.ts @@ -0,0 +1,143 @@ +import { describe, expect, it } from "vitest"; + +import { BME_MODULE_ADDRESS, deriveActMigrationSignals } from "@src/bme/act-migration-deriver"; +import type { DecodedBlock, DecodedEvent } from "@src/pipeline/decoded-block"; + +const BLOCK_TIME = new Date("2026-08-13T00:00:00Z"); + +const VAULT_FUNDED_EVENT_TYPE = "akash.bme.v1.EventVaultFunded"; +const EXECUTED_EVENT_TYPE = "akash.bme.v1.EventLedgerRecordExecuted"; +const PRICE_DATA_EVENT_TYPE = "akash.oracle.v1.EventPriceData"; +const SANDBOX_USDC_DENOM = "ibc/028CD1864059EEFB48A6048376165318E3E82C234390AE5A6D7B22001725B06E"; + +describe("deriveActMigrationSignals", () => { + it("flags any native BME event by prefix, including types the BME deriver does not parse", () => { + const signals = deriveActMigrationSignals(block({ blockEvents: [event(VAULT_FUNDED_EVENT_TYPE, { amount: '{"denom":"uakt","amount":"1000"}' })] })); + + expect(signals.hasNativeBmeEvent).toBe(true); + }); + + it("ignores synthetic legacy-indexer event types and unrelated events", () => { + const signals = deriveActMigrationSignals( + block({ + blockEvents: [event("indexer.bme.MigrationMinted", { amount: "1000" }), event("transfer", { amount: "1uakt" })] + }) + ); + + expect(signals.hasNativeBmeEvent).toBe(false); + }); + + it("keeps the last AKT/USD oracle price of the block for both uakt and akt denoms", () => { + const signals = deriveActMigrationSignals( + block({ + txEvents: [ + event(PRICE_DATA_EVENT_TYPE, priceDataAttributes({ denom: "uakt", price: "1.150000000000000000" })), + event(PRICE_DATA_EVENT_TYPE, priceDataAttributes({ denom: "akt", price: "1.160000000000000000" })) + ] + }) + ); + + expect(signals.lastAktUsdPrice).toBe("1.160000000000000000"); + }); + + it("ignores oracle prices for other pairs, malformed attributes and non-decimal prices", () => { + const signals = deriveActMigrationSignals( + block({ + txEvents: [ + event(PRICE_DATA_EVENT_TYPE, priceDataAttributes({ denom: "uusdc", price: "1.000000000000000000" })), + event(PRICE_DATA_EVENT_TYPE, { id: '{"denom":"uakt","base_denom":"eur"}', data: '{"price":"1.10","timestamp":"t"}' }), + event(PRICE_DATA_EVENT_TYPE, { id: "not-json", data: '{"price":"1.10","timestamp":"t"}' }), + event(PRICE_DATA_EVENT_TYPE, priceDataAttributes({ denom: "uakt", price: "-1.10" })) + ] + }) + ); + + expect(signals.lastAktUsdPrice).toBeNull(); + }); + + it("totals the BME module's burns and mints, keyed to the denoms the conversion moves", () => { + const signals = deriveActMigrationSignals( + block({ + blockEvents: [ + event("burn", { burner: BME_MODULE_ADDRESS, amount: "9034806372uakt" }), + event("coinbase", { minter: BME_MODULE_ADDRESS, amount: "5658593148uact" }), + event("burn", { burner: BME_MODULE_ADDRESS, amount: `19242170${SANDBOX_USDC_DENOM}` }), + event("burn", { burner: "akash1someoneelse", amount: "5uakt" }), + event("coinbase", { minter: BME_MODULE_ADDRESS, amount: "7uakt" }) + ] + }) + ); + + expect(signals.bankTotals).toEqual({ burnedUakt: 9034806372n, burnedUsdc: 19242170n, mintedUact: 5658593148n }); + }); + + it("splits comma-separated multi-coin burn and mint amounts across their denoms", () => { + const signals = deriveActMigrationSignals( + block({ + blockEvents: [ + event("burn", { burner: BME_MODULE_ADDRESS, amount: `100uakt,50${SANDBOX_USDC_DENOM}` }), + event("coinbase", { minter: BME_MODULE_ADDRESS, amount: "7uact,3uakt" }) + ] + }) + ); + + expect(signals.bankTotals).toEqual({ burnedUakt: 100n, burnedUsdc: 50n, mintedUact: 7n }); + }); + + it("marks blocks with executed ledger records so bank totals are not treated as conversion-only", () => { + const signals = deriveActMigrationSignals(block({ blockEvents: [event(EXECUTED_EVENT_TYPE, {})] })); + + expect(signals.hasLedgerExecutedEvent).toBe(true); + expect(signals.hasNativeBmeEvent).toBe(true); + }); + + it("skips events of failed transactions entirely", () => { + const signals = deriveActMigrationSignals( + block({ + code: 5, + txEvents: [event(VAULT_FUNDED_EVENT_TYPE, {}), event(PRICE_DATA_EVENT_TYPE, priceDataAttributes({ denom: "uakt", price: "1.15" }))] + }) + ); + + expect(signals.hasNativeBmeEvent).toBe(false); + expect(signals.lastAktUsdPrice).toBeNull(); + }); + + function priceDataAttributes(input: { denom: string; price: string }): Record { + return { + source: '"band"', + id: `{"denom":"${input.denom}","base_denom":"usd"}`, + data: `{"price":"${input.price}","timestamp":"2026-08-13T00:00:00Z"}` + }; + } + + function block(input: { height?: number; code?: number; txEvents?: DecodedEvent[]; blockEvents?: DecodedEvent[] }): DecodedBlock { + return { + height: input.height ?? 100, + datetime: BLOCK_TIME, + hash: Buffer.alloc(0), + parentHash: null, + proposerAddress: "P", + transactions: input.txEvents + ? [ + { + index: 0, + hash: Buffer.alloc(0), + code: input.code ?? 0, + gasUsed: 0, + gasWanted: 0, + fee: [], + messages: [], + events: input.txEvents, + signerAddresses: [] + } + ] + : [], + blockEvents: input.blockEvents ?? [] + }; + } + + function event(type: string, attributes: Record): DecodedEvent { + return { type, attributes }; + } +}); diff --git a/apps/chain-indexer/src/bme/act-migration-deriver.ts b/apps/chain-indexer/src/bme/act-migration-deriver.ts new file mode 100644 index 0000000000..49bb313484 --- /dev/null +++ b/apps/chain-indexer/src/bme/act-migration-deriver.ts @@ -0,0 +1,109 @@ +import { DENOM_MAPPING } from "@src/akash/denom"; +import { asString, parseJsonRecord } from "@src/akash/json"; +import { isDecString } from "@src/bme/bme-deriver"; +import { parseCoins } from "@src/pipeline/balance/coin-amount"; +import type { DecodedBlock, DecodedEvent } from "@src/pipeline/decoded-block"; + +/** Every native BME event type; only the chain's BME module emits under this prefix, and only from the upgrade block on. */ +const NATIVE_BME_EVENT_PREFIX = "akash.bme.v1."; +const LEDGER_RECORD_EXECUTED_EVENT_TYPE = "akash.bme.v1.EventLedgerRecordExecuted"; +const PRICE_DATA_EVENT_TYPE = "akash.oracle.v1.EventPriceData"; + +/** The x/bme module account — the burner/minter of every denom conversion — identical on every network. */ +export const BME_MODULE_ADDRESS = "akash1klpwzlvfnw7j8gtdd0cuu9vaw9ermsmd37sg55"; + +/** The axlUSDC IBC denoms the upgrade converted at par, derived from the single denom mapping so the two never drift. */ +export const IBC_USDC_DENOMS: readonly string[] = Array.from(DENOM_MAPPING.entries()) + .filter(([, baseDenom]) => baseDenom === "uusdc") + .map(([denom]) => denom); + +/** + * The bank movements of this block's denom conversions: what the BME module account burned and + * minted. Regular BME ledger executions burn and mint through the same account, so the totals only + * bind the conversion boundary when the block carries no `EventLedgerRecordExecuted`. + */ +export interface ActConversionBankTotals { + burnedUakt: bigint; + burnedUsdc: bigint; + mintedUact: bigint; +} + +export interface ActMigrationSignals { + hasNativeBmeEvent: boolean; + lastAktUsdPrice: string | null; + bankTotals: ActConversionBankTotals; + hasLedgerExecutedEvent: boolean; +} + +/** + * The BME upgrade converted every open escrow account in place without per-account events, so the + * conversion must be inferred from block content: the first native BME event marks the upgrade + * block, the BME module's burn/coinbase totals mark each drain block and how much it converted, and + * oracle `EventPriceData` supplies the drain rate. Oracle prices arrive via wasm transactions, so + * failed transactions are skipped; the conversion's bank events fire in the EndBlocker and are + * scanned there too. + */ +export function deriveActMigrationSignals(block: DecodedBlock): ActMigrationSignals { + const signals: ActMigrationSignals = { + hasNativeBmeEvent: false, + lastAktUsdPrice: null, + bankTotals: { burnedUakt: 0n, burnedUsdc: 0n, mintedUact: 0n }, + hasLedgerExecutedEvent: false + }; + + for (const tx of block.transactions) { + if (tx.code !== 0) { + continue; + } + scanEvents(tx.events, signals); + } + scanEvents(block.blockEvents, signals); + + return signals; +} + +function scanEvents(events: DecodedEvent[], signals: ActMigrationSignals): void { + for (const event of events) { + if (event.type.startsWith(NATIVE_BME_EVENT_PREFIX)) { + signals.hasNativeBmeEvent = true; + if (event.type === LEDGER_RECORD_EXECUTED_EVENT_TYPE) { + signals.hasLedgerExecutedEvent = true; + } + } else if (event.type === PRICE_DATA_EVENT_TYPE) { + const price = aktUsdPriceOf(event.attributes); + if (price !== null) { + signals.lastAktUsdPrice = price; + } + } else if (event.type === "burn" && event.attributes.burner === BME_MODULE_ADDRESS) { + for (const coin of parseBankCoins(event.attributes.amount)) { + if (coin.denom === "uakt") { + signals.bankTotals.burnedUakt += coin.amount; + } else if (IBC_USDC_DENOMS.includes(coin.denom)) { + signals.bankTotals.burnedUsdc += coin.amount; + } + } + } else if (event.type === "coinbase" && event.attributes.minter === BME_MODULE_ADDRESS) { + for (const coin of parseBankCoins(event.attributes.amount)) { + if (coin.denom === "uact") { + signals.bankTotals.mintedUact += coin.amount; + } + } + } + } +} + +function aktUsdPriceOf(attributes: Record): string | null { + const id = parseJsonRecord(attributes.id); + const denom = asString(id?.denom); + const baseDenom = asString(id?.base_denom); + if ((denom !== "uakt" && denom !== "akt") || baseDenom !== "usd") { + return null; + } + const data = parseJsonRecord(attributes.data); + const price = asString(data?.price); + return price !== null && isDecString(price) ? price : null; +} + +function parseBankCoins(amount: string | undefined) { + return amount === undefined ? [] : parseCoins(amount); +} diff --git a/apps/chain-indexer/src/bme/act-migration.service.spec.ts b/apps/chain-indexer/src/bme/act-migration.service.spec.ts new file mode 100644 index 0000000000..072039eaa0 --- /dev/null +++ b/apps/chain-indexer/src/bme/act-migration.service.spec.ts @@ -0,0 +1,457 @@ +import { toBech32 } from "@cosmjs/encoding"; +import { describe, expect, it } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import { ACT_MIGRATION_STREAMS, ActMigrationService } from "@src/bme/act-migration.service"; +import { ActMigrationQueue, ActMigrationState, Deployments, IndexerState, Leases } from "@src/db/schema"; +import type { DecodedBlock, DecodedEvent } from "@src/pipeline/decoded-block"; +import type { ChainDatabase, ChainTransaction } from "@src/providers/db.provider"; +import type { LoggerService } from "@src/providers/logging.provider"; + +const BLOCK_TIME = new Date("2026-08-13T00:00:00Z"); +const EARLIER_PRICE = "0.626310480000000000"; +const SAME_BLOCK_PRICE = "0.626185160000000000"; +const BME_MODULE_ADDRESS = "akash1klpwzlvfnw7j8gtdd0cuu9vaw9ermsmd37sg55"; +const SANDBOX_USDC_DENOM = "ibc/028CD1864059EEFB48A6048376165318E3E82C234390AE5A6D7B22001725B06E"; + +/** + * Real akash addresses whose raw 20-byte order (leading byte 0x00 < 0x08 < 0x10, the chain's + * `collections.Join(owner, dseq)` key order) is the reverse of their bech32-string order: bech32's + * charset maps 0x08's leading group to 'p' and 0x00's to 'q', so a string sort would order them + * `OWNER_BYTE_1` before `OWNER_BYTE_0`. These make a wrong string sort observable. + */ +const ownerWithLeadingByte = (leadingByte: number) => toBech32("akash", Uint8Array.from([leadingByte, ...new Array(19).fill(0)])); +const OWNER_BYTE_0 = ownerWithLeadingByte(0x00); +const OWNER_BYTE_1 = ownerWithLeadingByte(0x08); +const OWNER_BYTE_2 = ownerWithLeadingByte(0x10); + +describe(ActMigrationService.name, () => { + describe("segment", () => { + it("returns the whole batch as one segment when nothing triggers", async () => { + const { service } = setup(); + const blocks = [block({ height: 10 }), block({ height: 11 })]; + + const segments = await service.segment(blocks); + + expect(segments).toEqual([{ blocks, step: null, observations: { lastAktUsdPrice: null } }]); + }); + + it("fast-paths when the drained marker exists", async () => { + const { service } = setup({ markers: ["upgrade", "drained"] }); + const blocks = [block({ height: 10, blockEvents: [vaultFundedEvent(), ...conversionBankEvents("9uakt", "5uact")] })]; + + const segments = await service.segment(blocks); + + expect(segments[0].step).toBeNull(); + }); + + it("splits at the first native BME event and schedules the upgrade step", async () => { + const { service } = setup(); + const blocks = [block({ height: 10 }), block({ height: 11, blockEvents: [vaultFundedEvent()] }), block({ height: 12 })]; + + const segments = await service.segment(blocks); + + expect(segments).toHaveLength(2); + expect(segments[0].blocks).toEqual(blocks.slice(0, 2)); + expect(segments[0].step).toEqual(expect.objectContaining({ kind: "upgrade", height: 11 })); + expect(segments[1]).toEqual({ blocks: blocks.slice(2), step: null, observations: { lastAktUsdPrice: null } }); + }); + + it("schedules a drain at a conversion block using the latest price from strictly earlier blocks, not the block's own price", async () => { + const { service } = setup({ markers: ["upgrade"], queuePending: 30 }); + const blocks = [ + block({ height: 2552658, txEvents: [priceEvent(EARLIER_PRICE)] }), + block({ height: 2552659 }), + block({ height: 2552660, txEvents: [priceEvent(SAME_BLOCK_PRICE)], blockEvents: conversionBankEvents("9034806372uakt", "5658593148uact") }) + ]; + + const segments = await service.segment(blocks); + + expect(segments).toHaveLength(1); + expect(segments[0].step).toEqual( + expect.objectContaining({ + kind: "drain", + height: 2552660, + aktUsdRate: EARLIER_PRICE, + bankTotals: { burnedUakt: 9034806372n, burnedUsdc: 0n, mintedUact: 5658593148n } + }) + ); + }); + + it("throws when a conversion block appears before any oracle price was seen", async () => { + const { service } = setup({ markers: ["upgrade"], queuePending: 30 }); + const blocks = [block({ height: 20, blockEvents: conversionBankEvents("9uakt", "5uact") })]; + + await expect(service.segment(blocks)).rejects.toThrow("no AKT/USD oracle price was seen"); + }); + + it("does not schedule a drain without the burn and mint signature", async () => { + const { service } = setup({ markers: ["upgrade"], queuePending: 30, persistedPrice: EARLIER_PRICE }); + + const segments = await service.segment([block({ height: 20, txEvents: [priceEvent(SAME_BLOCK_PRICE)] })]); + + expect(segments[0].step).toBeNull(); + }); + + it("resumes mid-drain from the persisted price after a restart", async () => { + const { service } = setup({ markers: ["upgrade"], queuePending: 30, persistedPrice: EARLIER_PRICE }); + + const segments = await service.segment([block({ height: 20, blockEvents: conversionBankEvents("9uakt", "5uact") })]); + + expect(segments[0].step).toEqual(expect.objectContaining({ kind: "drain", height: 20, aktUsdRate: EARLIER_PRICE })); + }); + + it("does not retain planning observations that were never committed", async () => { + const { service } = setup({ markers: ["upgrade"], queuePending: 30 }); + await service.segment([block({ height: 20, txEvents: [priceEvent(EARLIER_PRICE)] })]); + + await expect(service.segment([block({ height: 21, blockEvents: conversionBankEvents("9uakt", "5uact") })])).rejects.toThrow("no AKT/USD oracle price"); + }); + + it("uses committed observations for later batches", async () => { + const { service } = setup({ markers: ["upgrade"], queuePending: 30 }); + const primer = (await service.segment([block({ height: 20, txEvents: [priceEvent(EARLIER_PRICE)] })]))[0]; + service.markCommitted(primer, null); + + const segments = await service.segment([block({ height: 21, blockEvents: conversionBankEvents("9uakt", "5uact") })]); + + expect(segments[0].step).toEqual(expect.objectContaining({ kind: "drain", height: 21, aktUsdRate: EARLIER_PRICE })); + }); + }); + + describe("applySegment", () => { + it("persists the segment's last observed price with a monotonic height guard", async () => { + const { service, tx, recorded } = setup(); + const segment = planlessSegment({ lastAktUsdPrice: { price: EARLIER_PRICE, height: 20 } }); + + const outcome = await service.applySegment(tx, segment); + + expect(outcome).toBeNull(); + expect(recorded.inserts.filter(insert => insert.table === ActMigrationState)).toEqual([ + expect.objectContaining({ rows: [{ id: 1, lastAktUsdPrice: EARLIER_PRICE, lastPriceHeight: 20 }] }) + ]); + }); + + it("does nothing when the segment carries no step and no price", async () => { + const { service, tx, recorded } = setup(); + + const outcome = await service.applySegment(tx, planlessSegment({})); + + expect(outcome).toBeNull(); + expect(recorded.inserts).toEqual([]); + expect(recorded.updates).toEqual([]); + }); + + it("converts axlUSDC deployments at par and seeds the drain queue in the chain's address-byte then dseq order", async () => { + const { service, tx, recorded } = setup({ + openDeployments: [ + { id: 3, denom: "uakt", balance: "10", dseq: "200", owner: OWNER_BYTE_0 }, + { id: 1, denom: "uusdc", balance: "19242170.5", dseq: "100", owner: OWNER_BYTE_2 }, + { id: 2, denom: "uakt", balance: "10", dseq: "50", owner: OWNER_BYTE_0 }, + { id: 4, denom: "uakt", balance: "10", dseq: "300", owner: OWNER_BYTE_1 } + ], + detectionResources: [ + { deploymentId: 3, gseq: 1, idx: 0, price: "5", priceDenom: "uakt" }, + { deploymentId: 1, gseq: 1, idx: 0, price: "5", priceDenom: SANDBOX_USDC_DENOM }, + { deploymentId: 2, gseq: 1, idx: 0, price: "5", priceDenom: "uakt" }, + { deploymentId: 4, gseq: 1, idx: 0, price: "5", priceDenom: "uakt" } + ] + }); + const step = upgradeStep({ height: 2552650, bankTotals: { burnedUakt: 0n, burnedUsdc: 19242170n, mintedUact: 19242170n } }); + + const outcome = await service.applySegment(tx, segmentWith(step)); + + expect(outcome).toEqual({ upgradeApplied: true, queueRemaining: 3, drained: false }); + expect(recorded.updates.find(update => update.table === Deployments)?.set).toEqual({ denom: "uact" }); + const queueInsert = recorded.inserts.find(insert => insert.table === ActMigrationQueue); + expect(queueInsert?.rows).toEqual([ + { position: 0, deploymentId: 2 }, + { position: 1, deploymentId: 3 }, + { position: 2, deploymentId: 4 } + ]); + }); + + it("marks the migration drained at the upgrade when no uakt deployment is open", async () => { + const { service, tx, recorded } = setup({ + openDeployments: [{ id: 1, denom: "uusdc", balance: "5", dseq: "100", owner: "akash1aaa" }], + detectionResources: [{ deploymentId: 1, gseq: 1, idx: 0, price: "5", priceDenom: SANDBOX_USDC_DENOM }] + }); + + const outcome = await service.applySegment(tx, segmentWith(upgradeStep({ height: 2552650 }))); + + expect(outcome).toEqual({ upgradeApplied: true, queueRemaining: 0, drained: true }); + expect(recorded.inserts.filter(insert => insert.table === IndexerState).map(insert => (insert.rows as Array<{ stream: string }>)[0].stream)).toEqual([ + ACT_MIGRATION_STREAMS.upgrade, + ACT_MIGRATION_STREAMS.drained + ]); + }); + + it("skips the upgrade work when another writer already claimed it", async () => { + const { service, tx, recorded } = setup({ claimRejects: [ACT_MIGRATION_STREAMS.upgrade] }); + + const outcome = await service.applySegment(tx, segmentWith(upgradeStep({ height: 2552650 }))); + + expect(outcome).toEqual({ upgradeApplied: true }); + expect(recorded.updates).toEqual([]); + }); + + it("drains until the computed totals equal the block's bank events, consuming closed slots along the way", async () => { + const { service, tx, recorded, logger } = setup({ + queuePending: 2, + queueSlots: [ + { position: 0, deploymentId: 12 }, + { position: 1, deploymentId: 11 } + ], + drainDeployments: [ + { id: 12, denom: "uakt", balance: "50", deposit: "50", withdrawnAmount: "0", blockRate: "0", closedHeight: 2552655 }, + { id: 11, denom: "uakt", balance: "100.5", deposit: "100", withdrawnAmount: "10", blockRate: "10", closedHeight: null } + ], + drainLeases: [{ deploymentId: 11, gseq: 1, oseq: 1, bseq: 0, providerAccountId: 7, price: "10", balance: "8", withdrawnAmount: "7" }], + remainingAfter: 0 + }); + const step = drainStep({ height: 2552660, aktUsdRate: "0.5", bankTotals: { burnedUakt: 108n, burnedUsdc: 0n, mintedUact: 54n } }); + + const outcome = await service.applySegment(tx, segmentWith(step)); + + expect(outcome).toEqual({ queueRemaining: 0, drained: true }); + const deploymentUpdate = recorded.updates.find(update => update.table === Deployments); + expect(deploymentUpdate?.set).toEqual({ denom: "uact", balance: "50.25", deposit: "50", withdrawnAmount: "5", blockRate: "5" }); + const leaseUpdate = recorded.updates.find(update => update.table === Leases); + expect(leaseUpdate?.set).toEqual({ denom: "uact", price: "5", balance: "4", withdrawnAmount: "3" }); + const queueUpdate = recorded.updates.find(update => update.table === ActMigrationQueue); + expect(queueUpdate?.set).toEqual({ convertedAtHeight: 2552660 }); + expect(logger.info).toHaveBeenCalledWith(expect.objectContaining({ event: "ACT_MIGRATION_DRAIN_APPLIED", converted: 1, skippedClosed: 1 })); + expect(recorded.inserts.filter(insert => insert.table === IndexerState).map(insert => (insert.rows as Array<{ stream: string }>)[0].stream)).toEqual([ + ACT_MIGRATION_STREAMS.drainAt(2552660), + ACT_MIGRATION_STREAMS.drained + ]); + }); + + it("aborts the block when converting the next queue entry would overshoot the chain's totals", async () => { + const { service, tx } = setup({ + queuePending: 1, + queueSlots: [{ position: 0, deploymentId: 11 }], + drainDeployments: [{ id: 11, denom: "uakt", balance: "100.5", deposit: "100", withdrawnAmount: "10", blockRate: "10", closedHeight: null }], + drainLeases: [] + }); + const step = drainStep({ height: 2552660, aktUsdRate: "0.5", bankTotals: { burnedUakt: 50n, burnedUsdc: 0n, mintedUact: 25n } }); + + await expect(service.applySegment(tx, segmentWith(step))).rejects.toThrow("overshoots the chain's totals"); + }); + + it("reports a shortfall when the queue runs out before the chain's totals are reached", async () => { + const { service, tx, logger } = setup({ + queuePending: 1, + queueSlots: [{ position: 0, deploymentId: 11 }], + drainDeployments: [{ id: 11, denom: "uakt", balance: "100.5", deposit: "100", withdrawnAmount: "10", blockRate: "10", closedHeight: null }], + drainLeases: [], + remainingAfter: 0, + extraEmptySlotReads: 1 + }); + const step = drainStep({ height: 2552660, aktUsdRate: "0.5", bankTotals: { burnedUakt: 9034806372n, burnedUsdc: 0n, mintedUact: 5658593148n } }); + + const outcome = await service.applySegment(tx, segmentWith(step)); + + expect(outcome).toEqual({ queueRemaining: 0, drained: true }); + expect(logger.warn).toHaveBeenCalledWith(expect.objectContaining({ event: "ACT_MIGRATION_DRAIN_SHORTFALL", converted: 1 })); + }); + + it("treats a conversion-signature block with an empty queue as already drained, even when mixed with ledger executions", async () => { + const { service, tx, recorded } = setup({ queuePending: 0 }); + const step = { ...drainStep({ height: 30, aktUsdRate: "0.5", bankTotals: { burnedUakt: 9n, burnedUsdc: 0n, mintedUact: 5n } }), validatable: false }; + + const outcome = await service.applySegment(tx, segmentWith(step)); + + expect(outcome).toEqual({ queueRemaining: 0, drained: true }); + expect(recorded.updates).toEqual([]); + }); + + it("aborts when a block mixes ledger executions with an unfinished drain", async () => { + const { service, tx } = setup({ queuePending: 5 }); + const step = { ...drainStep({ height: 30, aktUsdRate: "0.5", bankTotals: { burnedUakt: 9n, burnedUsdc: 0n, mintedUact: 5n } }), validatable: false }; + + await expect(service.applySegment(tx, segmentWith(step))).rejects.toThrow("cannot bind the conversion boundary"); + }); + + it("skips a drain height another writer already claimed without consuming queue slots", async () => { + const { service, tx, recorded } = setup({ claimRejects: [ACT_MIGRATION_STREAMS.drainAt(2552660)] }); + + const outcome = await service.applySegment(tx, segmentWith(drainStep({ height: 2552660, aktUsdRate: "0.5" }))); + + expect(outcome).toBeNull(); + expect(recorded.updates).toEqual([]); + }); + }); + + function vaultFundedEvent(): DecodedEvent { + return { type: "akash.bme.v1.EventVaultFunded", attributes: { amount: '{"denom":"uakt","amount":"1000"}' } }; + } + + function conversionBankEvents(burned: string, minted: string): DecodedEvent[] { + return [ + { type: "burn", attributes: { burner: BME_MODULE_ADDRESS, amount: burned } }, + { type: "coinbase", attributes: { minter: BME_MODULE_ADDRESS, amount: minted } } + ]; + } + + function priceEvent(price: string): DecodedEvent { + return { + type: "akash.oracle.v1.EventPriceData", + attributes: { source: '"band"', id: '{"denom":"akt","base_denom":"usd"}', data: `{"price":"${price}","timestamp":"t"}` } + }; + } + + function block(input: { height: number; txEvents?: DecodedEvent[]; blockEvents?: DecodedEvent[] }): DecodedBlock { + return { + height: input.height, + datetime: BLOCK_TIME, + hash: Buffer.alloc(0), + parentHash: null, + proposerAddress: "P", + transactions: input.txEvents + ? [ + { + index: 0, + hash: Buffer.alloc(0), + code: 0, + gasUsed: 0, + gasWanted: 0, + fee: [], + messages: [], + events: input.txEvents, + signerAddresses: [] + } + ] + : [], + blockEvents: input.blockEvents ?? [] + }; + } + + function planlessSegment(observations: { lastAktUsdPrice?: { price: string; height: number } }) { + return { + blocks: [block({ height: 20 })], + step: null, + observations: { lastAktUsdPrice: observations.lastAktUsdPrice ?? null } + }; + } + + function segmentWith(step: Parameters[1]["step"]) { + return { blocks: [block({ height: step?.height ?? 20 })], step, observations: { lastAktUsdPrice: null } }; + } + + function upgradeStep(input: { height: number; bankTotals?: { burnedUakt: bigint; burnedUsdc: bigint; mintedUact: bigint } }) { + return { + kind: "upgrade" as const, + height: input.height, + bankTotals: input.bankTotals ?? { burnedUakt: 0n, burnedUsdc: 0n, mintedUact: 0n }, + validatable: true + }; + } + + function drainStep(input: { height: number; aktUsdRate: string; bankTotals?: { burnedUakt: bigint; burnedUsdc: bigint; mintedUact: bigint } }) { + return { + kind: "drain" as const, + height: input.height, + aktUsdRate: input.aktUsdRate, + bankTotals: input.bankTotals ?? { burnedUakt: 0n, burnedUsdc: 0n, mintedUact: 0n }, + validatable: true + }; + } + + function setup(input?: { + markers?: Array<"upgrade" | "drained">; + queuePending?: number; + persistedPrice?: string; + claimRejects?: string[]; + openDeployments?: Array>; + detectionResources?: Array>; + queueSlots?: Array<{ position: number; deploymentId: number }>; + drainDeployments?: Array>; + drainLeases?: Array>; + remainingAfter?: number; + extraEmptySlotReads?: number; + }) { + const markerRows = (input?.markers ?? []).map(kind => ({ stream: ACT_MIGRATION_STREAMS[kind] })); + const stateRows = input?.persistedPrice === undefined ? [] : [{ id: 1, lastAktUsdPrice: input.persistedPrice, lastPriceHeight: 1 }]; + + const dbFake = { + select: () => ({ + from: (table: unknown) => { + if (table === IndexerState) return query(markerRows); + if (table === ActMigrationQueue) return query([{ remaining: input?.queuePending ?? 1 }]); + if (table === ActMigrationState) return query(stateRows); + return query([]); + } + }) + }; + + const recorded = { + inserts: [] as Array<{ table: unknown; rows: unknown }>, + updates: [] as Array<{ table: unknown; set: Record; where: unknown }> + }; + const queueSelects: unknown[][] = [[{ remaining: input?.queuePending ?? 0 }]]; + if (input?.queueSlots) { + queueSelects.push(input.queueSlots); + for (let i = 0; i < (input?.extraEmptySlotReads ?? 0); i++) { + queueSelects.push([]); + } + queueSelects.push([{ remaining: input?.remainingAfter ?? 0 }]); + } + const txSelects = new Map([ + [ActMigrationQueue, queueSelects], + [Deployments, [input?.openDeployments ?? input?.drainDeployments ?? []]], + [Leases, [input?.drainLeases ?? [], [], []]], + [DeploymentGroupResourcesToken, [input?.detectionResources ?? [], [], [], [], []]] + ]); + + const txFake = { + select: () => ({ + from: (table: unknown) => { + const key = txSelects.has(table) ? table : DeploymentGroupResourcesToken; + const queues = txSelects.get(key); + const rows = queues && queues.length > 0 ? queues.shift() : []; + return query(rows ?? []); + } + }), + insert: (table: unknown) => ({ + values: (rows: unknown) => { + recorded.inserts.push({ table, rows: Array.isArray(rows) ? rows : [rows] }); + const claimStream = table === IndexerState && !Array.isArray(rows) ? (rows as { stream: string }).stream : null; + const claimResult = claimStream !== null && input?.claimRejects?.includes(claimStream) ? [] : [rows]; + return Object.assign(Promise.resolve(), { + onConflictDoNothing: () => ({ returning: () => Promise.resolve(claimResult) }), + onConflictDoUpdate: () => Promise.resolve() + }); + } + }), + update: (table: unknown) => ({ + set: (set: Record) => ({ + where: (where: unknown) => { + recorded.updates.push({ table, set, where }); + return Object.assign(Promise.resolve(), { returning: () => Promise.resolve([]) }); + } + }) + }) + }; + + const logger = mock(); + const service = new ActMigrationService(dbFake as unknown as ChainDatabase, logger); + return { service, logger, tx: txFake as unknown as ChainTransaction, recorded }; + } + + const DeploymentGroupResourcesToken = Symbol("group_resources"); + + function query(rows: unknown) { + const chain: Record = {}; + const self = () => chain; + Object.assign(chain, { + innerJoin: self, + where: self, + orderBy: self, + limit: self, + for: self, + then: (resolve: (rows: unknown) => unknown, reject?: (error: unknown) => unknown) => Promise.resolve(rows).then(resolve, reject) + }); + return chain; + } +}); diff --git a/apps/chain-indexer/src/bme/act-migration.service.ts b/apps/chain-indexer/src/bme/act-migration.service.ts new file mode 100644 index 0000000000..66a262532f --- /dev/null +++ b/apps/chain-indexer/src/bme/act-migration.service.ts @@ -0,0 +1,602 @@ +import { fromBech32 } from "@cosmjs/encoding"; +import { and, eq, gt, inArray, isNotNull, isNull, or, sql } from "drizzle-orm"; +import { inject, singleton } from "tsyringe"; + +import { + conversionBankContribution, + convertDeploymentAmounts, + convertLeaseAmounts, + convertPriceAmount, + parseRate, + RATE_ONE +} from "@src/bme/act-migration-convert"; +import type { ActConversionBankTotals } from "@src/bme/act-migration-deriver"; +import { deriveActMigrationSignals, IBC_USDC_DENOMS } from "@src/bme/act-migration-deriver"; +import { + Accounts, + ActMigrationQueue, + ActMigrationState, + Bids, + DeploymentGroupResources, + DeploymentGroups, + Deployments, + IndexerState, + Leases +} from "@src/db/schema"; +import type { DecodedBlock } from "@src/pipeline/decoded-block"; +import type { ChainDatabase, ChainTransaction } from "@src/providers/db.provider"; +import { CHAIN_DB } from "@src/providers/db.provider"; +import { LoggerService } from "@src/providers/logging.provider"; + +export const ACT_MIGRATION_STREAMS = { + upgrade: "act-migration:upgrade", + drained: "act-migration:drained", + drainAt: (height: number) => `act-migration:drain:${height}` +} as const; + +/** Queue rows are read in chunks until the block's burn/mint totals are reached. */ +const DRAIN_CHUNK_SIZE = 500; + +export type ActMigrationStep = + | { kind: "upgrade"; height: number; bankTotals: ActConversionBankTotals; validatable: boolean } + | { kind: "drain"; height: number; aktUsdRate: string; bankTotals: ActConversionBankTotals; validatable: boolean }; + +interface SegmentObservations { + lastAktUsdPrice: { price: string; height: number } | null; +} + +export interface ActMigrationSegment { + blocks: DecodedBlock[]; + step: ActMigrationStep | null; + observations: SegmentObservations; +} + +export interface ActMigrationOutcome { + upgradeApplied?: boolean; + queueRemaining?: number; + drained?: boolean; +} + +/** + * In-place escrow denom conversion mirroring what the BME network upgrade did to chain state + * without per-account events. At the upgrade block (first native BME event) the chain converted + * axlUSDC deployments to uact at par and queued every open uakt deployment; its deployment + * EndBlocker then drained the queue over the following blocks, each block converting at the + * oracle's aggregated AKT/USD price stored at the previous block. How many deployments each block + * drained is chain-version-dependent (the sandbox RC drained everything at once, v2.0.0 caps at 50 + * per block), so the drain follows the observable record instead: each block's BME-module + * burn/coinbase totals say exactly how much was converted, and queue entries are consumed in the + * chain's order until the computed totals match them. A rate or ordering error therefore cannot + * corrupt silently — the totals fail to bind and the block aborts. The rate is the latest + * `EventPriceData` from strictly earlier blocks, matching the one-block lag of the oracle's stored + * aggregate. Every step claims an `indexer_state` marker, so replays and concurrent writers are + * exactly-once. + * + * Correctness inherits the pipeline's ordering contract (blocks processed in height order from a + * deployment's creation), and the drain additionally assumes full history: a partial-window + * backfill carries only a subset of the chain's queue, which surfaces as a logged shortfall. A + * module replay that rebuilds deployments from scratch must clear the `act-migration:*` markers + * and queue together with the module's rows, or replayed pre-upgrade rows would keep their + * creation-era denoms. + */ +@singleton() +export class ActMigrationService { + readonly #db: ChainDatabase; + readonly #logger: LoggerService; + #loaded = false; + #upgradeApplied = false; + #drained = false; + #lastAktUsdPrice: string | null = null; + + constructor(@inject(CHAIN_DB) db: ChainDatabase, @inject(LoggerService) logger: LoggerService) { + this.#db = db; + this.#logger = logger; + this.#logger.setContext("ACT_MIGRATION"); + } + + /** + * Splits a contiguous batch so each conversion step lands at the end of its own segment: the + * committer commits blocks up to and including the step block, applies the conversion in that + * same transaction, and only then processes later blocks — whose settlements must already see + * converted state. Planning never mutates the durable in-memory state; `markCommitted` folds a + * segment in only after its transaction committed, so a failed commit replans from scratch. + */ + async segment(blocks: DecodedBlock[]): Promise { + if (blocks.length === 0) { + return []; + } + await this.#ensureLoaded(); + if (this.#drained) { + return [{ blocks, step: null, observations: { lastAktUsdPrice: null } }]; + } + + const segments: ActMigrationSegment[] = []; + let upgradeApplied = this.#upgradeApplied; + let lastPrice = this.#lastAktUsdPrice; + let segmentStart = 0; + let observations: SegmentObservations = { lastAktUsdPrice: null }; + + blocks.forEach((block, index) => { + const signals = deriveActMigrationSignals(block); + + let step: ActMigrationStep | null = null; + if (!upgradeApplied && signals.hasNativeBmeEvent) { + upgradeApplied = true; + step = { kind: "upgrade", height: block.height, bankTotals: signals.bankTotals, validatable: !signals.hasLedgerExecutedEvent }; + } else if (upgradeApplied && signals.bankTotals.burnedUakt > 0n && signals.bankTotals.mintedUact > 0n) { + if (lastPrice === null) { + throw new Error( + `Block ${block.height} converted escrow (burned ${signals.bankTotals.burnedUakt}uakt) but no AKT/USD oracle price was seen in earlier blocks` + ); + } + step = { kind: "drain", height: block.height, aktUsdRate: lastPrice, bankTotals: signals.bankTotals, validatable: !signals.hasLedgerExecutedEvent }; + } + + if (signals.lastAktUsdPrice !== null) { + lastPrice = signals.lastAktUsdPrice; + observations.lastAktUsdPrice = { price: signals.lastAktUsdPrice, height: block.height }; + } + + if (step !== null) { + segments.push({ blocks: blocks.slice(segmentStart, index + 1), step, observations }); + segmentStart = index + 1; + observations = { lastAktUsdPrice: null }; + } + }); + + if (segmentStart < blocks.length) { + segments.push({ blocks: blocks.slice(segmentStart), step: null, observations }); + } + return segments; + } + + async applySegment(tx: ChainTransaction, segment: ActMigrationSegment): Promise { + if (this.#drained || (segment.step === null && segment.observations.lastAktUsdPrice === null)) { + return null; + } + + if (segment.observations.lastAktUsdPrice !== null) { + await this.#persistLastPrice(tx, segment.observations.lastAktUsdPrice); + } + if (segment.step === null) { + return null; + } + + return segment.step.kind === "upgrade" ? this.#applyUpgrade(tx, segment.step) : this.#applyDrain(tx, segment.step); + } + + /** Fold a segment into the durable in-memory state only after its transaction committed. */ + markCommitted(segment: ActMigrationSegment, outcome: ActMigrationOutcome | null): void { + if (segment.observations.lastAktUsdPrice !== null) { + this.#lastAktUsdPrice = segment.observations.lastAktUsdPrice.price; + } + if (segment.step?.kind === "upgrade" || outcome?.upgradeApplied) { + this.#upgradeApplied = true; + } + if (outcome?.drained) { + this.#drained = true; + } + } + + /** + * The upgrade block's conversion: axlUSDC deployments convert to uact at par immediately, and + * every open uakt deployment enters the drain queue in the chain's order — lexicographic owner + * address, then dseq. Denoms are detected from the first non-zero group resource price, exactly + * like the chain's `DetectDenom`; deployments whose groups carry no priced resources are skipped + * on chain and stay unconverted here too. + */ + async #applyUpgrade(tx: ChainTransaction, step: ActMigrationStep): Promise { + const claimed = await this.#claimMarker(tx, ACT_MIGRATION_STREAMS.upgrade, step.height); + if (!claimed) { + this.#logger.info({ event: "ACT_MIGRATION_UPGRADE_ALREADY_APPLIED", height: step.height }); + return { upgradeApplied: true }; + } + + const open = await tx + .select({ + id: Deployments.id, + denom: Deployments.denom, + balance: Deployments.balance, + dseq: Deployments.dseq, + owner: Accounts.address + }) + .from(Deployments) + .innerJoin(Accounts, eq(Deployments.ownerAccountId, Accounts.id)) + .where(isNull(Deployments.closedHeight)); + const openIds = open.map(deployment => deployment.id); + + const detected = await this.#detectDenoms(tx, openIds); + const usdc = open.filter(deployment => { + const detectedDenom = detected.get(deployment.id); + return detectedDenom !== undefined && IBC_USDC_DENOMS.includes(detectedDenom); + }); + const uakt = orderByChainDrainSequence(open.filter(deployment => detected.get(deployment.id) === "uakt")); + + const usdcIds = usdc.map(deployment => deployment.id); + const converted = await this.#renameUsdcToAct(tx, usdcIds); + + if (uakt.length > 0) { + await tx.insert(ActMigrationQueue).values(uakt.map((deployment, position) => ({ position, deploymentId: deployment.id }))); + } + + const drained = await this.#claimDrainedIfEmpty(tx, step.height, uakt.length); + + await this.#validateUsdcTotals(tx, step, usdc, usdcIds); + + this.#logger.info({ + event: "ACT_MIGRATION_UPGRADE_APPLIED", + height: step.height, + usdcDeployments: usdcIds.length, + usdcLeases: converted.leases, + queueSeeded: uakt.length + }); + + return { upgradeApplied: true, queueRemaining: uakt.length, drained }; + } + + /** + * One drain block: consume queue entries in order — converting the open uakt ones at this block's + * rate, spending the slot of any that closed while queued — until the computed burn/mint totals + * equal the block's bank events. Overshooting the totals means the rate or queue order is wrong + * and aborts the transaction; running out of queue first means this database carries only part of + * the chain's history (a partial-window backfill) and is reported as a shortfall. + */ + async #applyDrain(tx: ChainTransaction, step: ActMigrationStep & { kind: "drain" }): Promise { + const claimed = await this.#claimMarker(tx, ACT_MIGRATION_STREAMS.drainAt(step.height), step.height); + if (!claimed) { + this.#logger.info({ event: "ACT_MIGRATION_DRAIN_ALREADY_APPLIED", height: step.height }); + return null; + } + + const [{ remaining: pending }] = await tx + .select({ remaining: sql`COUNT(*)::int` }) + .from(ActMigrationQueue) + .where(isNull(ActMigrationQueue.convertedAtHeight)); + if (await this.#claimDrainedIfEmpty(tx, step.height, pending)) { + return { queueRemaining: 0, drained: true }; + } + + if (!step.validatable) { + throw new Error(`Block ${step.height} mixes BME ledger executions with an unfinished denom drain; its bank totals cannot bind the conversion boundary`); + } + + const rate = parseRate(step.aktUsdRate); + let burned = 0n; + let minted = 0n; + let convertedCount = 0; + let skippedClosed = 0; + const consumedPositions: number[] = []; + let cutReached = false; + let positionCursor = -1; + + while (!cutReached) { + const slots = await tx + .select({ position: ActMigrationQueue.position, deploymentId: ActMigrationQueue.deploymentId }) + .from(ActMigrationQueue) + .where(and(isNull(ActMigrationQueue.convertedAtHeight), gt(ActMigrationQueue.position, positionCursor))) + .orderBy(ActMigrationQueue.position) + .limit(DRAIN_CHUNK_SIZE) + .for("update"); + if (slots.length === 0) { + break; + } + positionCursor = slots[slots.length - 1].position; + + const deploymentIds = slots.map(slot => slot.deploymentId); + const [deployments, leases] = await Promise.all([ + tx.select().from(Deployments).where(inArray(Deployments.id, deploymentIds)).for("update"), + tx + .select() + .from(Leases) + .where(and(inArray(Leases.deploymentId, deploymentIds), isNull(Leases.closedHeight), eq(Leases.denom, "uakt"))) + ]); + const deploymentById = new Map(deployments.map(deployment => [deployment.id, deployment])); + const leasesByDeployment = new Map(); + for (const lease of leases) { + leasesByDeployment.set(lease.deploymentId, [...(leasesByDeployment.get(lease.deploymentId) ?? []), lease]); + } + + for (const slot of slots) { + const deployment = deploymentById.get(slot.deploymentId); + if (!deployment || deployment.closedHeight !== null || deployment.denom !== "uakt") { + skippedClosed += 1; + consumedPositions.push(slot.position); + continue; + } + + const deploymentLeases = leasesByDeployment.get(slot.deploymentId) ?? []; + const contribution = conversionBankContribution({ balance: deployment.balance, leaseBalances: deploymentLeases.map(lease => lease.balance) }, rate); + if (burned + contribution.burned > step.bankTotals.burnedUakt || minted + contribution.minted > step.bankTotals.mintedUact) { + throw new Error( + `ACT drain at block ${step.height} overshoots the chain's totals: queue position ${slot.position} would push ` + + `burned to ${burned + contribution.burned}/${step.bankTotals.burnedUakt}uakt and minted to ` + + `${minted + contribution.minted}/${step.bankTotals.mintedUact}uact — rate ${step.aktUsdRate} or queue order is wrong` + ); + } + + burned += contribution.burned; + minted += contribution.minted; + + await tx + .update(Deployments) + .set({ denom: "uact", ...convertDeploymentAmounts(deployment, rate) }) + .where(eq(Deployments.id, deployment.id)); + + for (const lease of deploymentLeases) { + await tx + .update(Leases) + .set({ denom: "uact", ...convertLeaseAmounts(lease, rate) }) + .where( + and( + eq(Leases.deploymentId, lease.deploymentId), + eq(Leases.gseq, lease.gseq), + eq(Leases.oseq, lease.oseq), + eq(Leases.bseq, lease.bseq), + eq(Leases.providerAccountId, lease.providerAccountId) + ) + ); + } + + await this.#convertBidPrices(tx, deployment.id, "uakt", rate); + await this.#convertGroupResourcePrices(tx, deployment.id, ["uakt"], rate); + convertedCount += 1; + consumedPositions.push(slot.position); + + cutReached = burned === step.bankTotals.burnedUakt && minted === step.bankTotals.mintedUact; + if (cutReached) { + break; + } + } + } + + if (consumedPositions.length > 0) { + await tx.update(ActMigrationQueue).set({ convertedAtHeight: step.height }).where(inArray(ActMigrationQueue.position, consumedPositions)); + } + + const [{ remaining }] = await tx + .select({ remaining: sql`COUNT(*)::int` }) + .from(ActMigrationQueue) + .where(isNull(ActMigrationQueue.convertedAtHeight)); + const drained = await this.#claimDrainedIfEmpty(tx, step.height, remaining); + + const logPayload = { + event: cutReached ? "ACT_MIGRATION_DRAIN_APPLIED" : "ACT_MIGRATION_DRAIN_SHORTFALL", + height: step.height, + aktUsdRate: step.aktUsdRate, + converted: convertedCount, + skippedClosed, + remaining, + burnedUakt: burned.toString(), + mintedUact: minted.toString(), + eventBurnedUakt: step.bankTotals.burnedUakt.toString(), + eventMintedUact: step.bankTotals.mintedUact.toString() + }; + if (cutReached) { + this.#logger.info(logPayload); + } else { + this.#logger.warn(logPayload); + } + + return { queueRemaining: remaining, drained }; + } + + /** axlUSDC converts at par: pure denom renames, amounts untouched — multiplying by one is exact. */ + async #renameUsdcToAct(tx: ChainTransaction, usdcIds: number[]): Promise<{ leases: number }> { + if (usdcIds.length > 0) { + await tx.update(Deployments).set({ denom: "uact" }).where(inArray(Deployments.id, usdcIds)); + await this.#convertBidPrices(tx, usdcIds, IBC_USDC_DENOMS, RATE_ONE); + for (const deploymentId of usdcIds) { + await this.#convertGroupResourcePrices(tx, deploymentId, IBC_USDC_DENOMS, RATE_ONE); + } + } + + const orphanOwnerFilter = or( + usdcIds.length > 0 ? inArray(Leases.deploymentId, usdcIds) : sql`false`, + inArray(Leases.deploymentId, tx.select({ id: Deployments.id }).from(Deployments).where(isNotNull(Deployments.closedHeight))) + ); + const renamedLeases = await tx + .update(Leases) + .set({ denom: "uact" }) + .where(and(eq(Leases.denom, "uusdc"), isNull(Leases.closedHeight), orphanOwnerFilter)) + .returning({ deploymentId: Leases.deploymentId }); + + return { leases: renamedLeases.length }; + } + + async #convertBidPrices(tx: ChainTransaction, deploymentIds: number | number[], fromDenoms: string | readonly string[], rate: bigint): Promise { + const idFilter = Array.isArray(deploymentIds) ? inArray(Bids.deploymentId, deploymentIds) : eq(Bids.deploymentId, deploymentIds as number); + const denomFilter = typeof fromDenoms === "string" ? eq(Bids.denom, fromDenoms) : inArray(Bids.denom, [...fromDenoms]); + const bids = await tx + .select() + .from(Bids) + .where(and(idFilter, denomFilter, inArray(Bids.state, ["open", "active"]))); + + for (const bid of bids) { + await tx + .update(Bids) + .set({ denom: "uact", price: convertPriceAmount(bid.price, rate) }) + .where( + and( + eq(Bids.deploymentId, bid.deploymentId), + eq(Bids.gseq, bid.gseq), + eq(Bids.oseq, bid.oseq), + eq(Bids.bseq, bid.bseq), + eq(Bids.providerAccountId, bid.providerAccountId) + ) + ); + } + } + + async #convertGroupResourcePrices(tx: ChainTransaction, deploymentId: number, fromDenoms: readonly string[], rate: bigint): Promise { + const resources = await tx + .select({ + deploymentGroupId: DeploymentGroupResources.deploymentGroupId, + idx: DeploymentGroupResources.idx, + price: DeploymentGroupResources.price, + priceDenom: DeploymentGroupResources.priceDenom + }) + .from(DeploymentGroupResources) + .innerJoin(DeploymentGroups, eq(DeploymentGroupResources.deploymentGroupId, DeploymentGroups.id)) + .where( + and( + eq(DeploymentGroups.deploymentId, deploymentId), + inArray(DeploymentGroups.state, ["open", "paused"]), + inArray(DeploymentGroupResources.priceDenom, [...fromDenoms]) + ) + ); + + for (const resource of resources) { + await tx + .update(DeploymentGroupResources) + .set({ priceDenom: "uact", price: convertPriceAmount(resource.price, rate) }) + .where(and(eq(DeploymentGroupResources.deploymentGroupId, resource.deploymentGroupId), eq(DeploymentGroupResources.idx, resource.idx))); + } + } + + /** Mirrors the chain's `DetectDenom`: the first non-zero resource price by group then resource order names the deployment's denom. */ + async #detectDenoms(tx: ChainTransaction, deploymentIds: number[]): Promise> { + if (deploymentIds.length === 0) { + return new Map(); + } + const resources = await tx + .select({ + deploymentId: DeploymentGroups.deploymentId, + gseq: DeploymentGroups.gseq, + idx: DeploymentGroupResources.idx, + price: DeploymentGroupResources.price, + priceDenom: DeploymentGroupResources.priceDenom + }) + .from(DeploymentGroupResources) + .innerJoin(DeploymentGroups, eq(DeploymentGroupResources.deploymentGroupId, DeploymentGroups.id)) + .where(inArray(DeploymentGroups.deploymentId, deploymentIds)) + .orderBy(DeploymentGroups.deploymentId, DeploymentGroups.gseq, DeploymentGroupResources.idx); + + const detected = new Map(); + for (const resource of resources) { + if (!detected.has(resource.deploymentId) && resource.price !== "0" && Number(resource.price) !== 0) { + detected.set(resource.deploymentId, resource.priceDenom); + } + } + return detected; + } + + /** + * Advisory: the chain's upgrade also converted orphaned escrow accounts of closed deployments, + * which this model does not track, so the event totals may exceed the computed ones. A shortfall + * the other way would mean the conversion model is wrong. + */ + async #validateUsdcTotals(tx: ChainTransaction, step: ActMigrationStep, usdc: Array<{ id: number; balance: string }>, usdcIds: number[]): Promise { + if (!step.validatable) { + return; + } + + let expectedBurned = 0n; + let expectedMinted = 0n; + if (usdcIds.length > 0) { + const usdcLeases = await tx + .select({ deploymentId: Leases.deploymentId, balance: Leases.balance }) + .from(Leases) + .where(and(inArray(Leases.deploymentId, usdcIds), isNull(Leases.closedHeight), eq(Leases.denom, "uact"))); + const leaseBalancesByDeployment = new Map(); + for (const lease of usdcLeases) { + leaseBalancesByDeployment.set(lease.deploymentId, [...(leaseBalancesByDeployment.get(lease.deploymentId) ?? []), lease.balance]); + } + for (const deployment of usdc) { + const contribution = conversionBankContribution( + { balance: deployment.balance, leaseBalances: leaseBalancesByDeployment.get(deployment.id) ?? [] }, + RATE_ONE + ); + expectedBurned += contribution.burned; + expectedMinted += contribution.minted; + } + } + + if (step.bankTotals.burnedUsdc !== expectedBurned || step.bankTotals.mintedUact !== expectedMinted) { + this.#logger.warn({ + event: "ACT_MIGRATION_USDC_TOTALS_MISMATCH", + height: step.height, + expectedBurnedUsdc: expectedBurned.toString(), + expectedMintedUact: expectedMinted.toString(), + eventBurnedUsdc: step.bankTotals.burnedUsdc.toString(), + eventMintedUact: step.bankTotals.mintedUact.toString() + }); + } + } + + async #persistLastPrice(tx: ChainTransaction, lastPrice: { price: string; height: number }): Promise { + await tx + .insert(ActMigrationState) + .values({ id: 1, lastAktUsdPrice: lastPrice.price, lastPriceHeight: lastPrice.height }) + .onConflictDoUpdate({ + target: ActMigrationState.id, + set: { lastAktUsdPrice: lastPrice.price, lastPriceHeight: lastPrice.height }, + setWhere: sql`EXCLUDED.last_price_height >= ${ActMigrationState.lastPriceHeight}` + }); + } + + async #claimMarker(tx: ChainTransaction, stream: string, height: number): Promise { + const claimed = await tx.insert(IndexerState).values({ stream, lastHeight: height, updatedAt: new Date() }).onConflictDoNothing().returning(); + return claimed.length > 0; + } + + async #claimDrainedIfEmpty(tx: ChainTransaction, height: number, remainingCount: number): Promise { + if (remainingCount !== 0) { + return false; + } + await this.#claimMarker(tx, ACT_MIGRATION_STREAMS.drained, height); + return true; + } + + /** + * The markers are authoritative for the upgrade and drained states; the last oracle price + * re-derives from the singleton state row, written in the same transactions as the blocks that + * produced it, so a restart resumes mid-drain exactly where it left off. + */ + async #ensureLoaded(): Promise { + if (this.#loaded) { + return; + } + + const markers = await this.#db + .select({ stream: IndexerState.stream }) + .from(IndexerState) + .where(inArray(IndexerState.stream, [ACT_MIGRATION_STREAMS.upgrade, ACT_MIGRATION_STREAMS.drained])); + this.#upgradeApplied = markers.some(marker => marker.stream === ACT_MIGRATION_STREAMS.upgrade); + this.#drained = markers.some(marker => marker.stream === ACT_MIGRATION_STREAMS.drained); + + if (!this.#drained) { + if (this.#upgradeApplied) { + const [{ remaining }] = await this.#db + .select({ remaining: sql`COUNT(*)::int` }) + .from(ActMigrationQueue) + .where(isNull(ActMigrationQueue.convertedAtHeight)); + this.#drained = remaining === 0; + } + + const [state] = await this.#db.select().from(ActMigrationState).where(eq(ActMigrationState.id, 1)); + this.#lastAktUsdPrice = state?.lastAktUsdPrice ?? null; + } + + this.#loaded = true; + } +} + +/** + * The chain seeds `pendingDenomMigrations` via `collections.Join(owner, dseq)`, iterating raw + * AccAddress bytes then dseq. Bech32-string order does not reproduce that byte order — the bech32 + * charset is not ASCII-monotonic — so the drain queue is ordered by the decoded address bytes to + * match the exact sequence the chain converts deployments in across drain blocks. + */ +function orderByChainDrainSequence(deployments: T[]): T[] { + return deployments + .map(deployment => ({ deployment, ownerBytes: fromBech32(deployment.owner).data })) + .sort((left, right) => Buffer.compare(left.ownerBytes, right.ownerBytes) || compareDseq(left.deployment.dseq, right.deployment.dseq)) + .map(entry => entry.deployment); +} + +/** Postgres normalizes numeric literals, so dseq order must compare numerically, matching the chain's uint64 key. */ +function compareDseq(a: string, b: string): number { + const left = BigInt(a); + const right = BigInt(b); + return left < right ? -1 : left > right ? 1 : 0; +} diff --git a/apps/chain-indexer/src/bme/bme-deriver.ts b/apps/chain-indexer/src/bme/bme-deriver.ts index 7e49cdd10e..fc7bcd74ea 100644 --- a/apps/chain-indexer/src/bme/bme-deriver.ts +++ b/apps/chain-indexer/src/bme/bme-deriver.ts @@ -283,6 +283,6 @@ function isMintStatus(value: string | null): value is BmeMintStatus { return value !== null && MINT_STATUSES.has(value); } -function isDecString(value: string): boolean { +export function isDecString(value: string): boolean { return /^\d+(\.\d+)?$/.test(value); } diff --git a/apps/chain-indexer/src/db/schema.ts b/apps/chain-indexer/src/db/schema.ts index 254ab8d012..2586d612dd 100644 --- a/apps/chain-indexer/src/db/schema.ts +++ b/apps/chain-indexer/src/db/schema.ts @@ -648,6 +648,38 @@ export const BmeCanceledRecords = akashSchema.table( t => [primaryKey({ columns: [t.recordHeight, t.sequence, t.denom, t.toDenom, t.source] }), index("bme_canceled_records_height_idx").on(t.height)] ); +/** + * The BME upgrade's deferred uakt→uact conversion queue, mirroring the chain's + * `pendingDenomMigrations`: seeded at the upgrade block with every open uakt deployment in the + * chain's drain order — lexicographic owner address, then dseq — and consumed at up to 50 rows per + * eligible block, exactly as the deployment EndBlocker drains it. `converted_at_height` marks a + * consumed slot even when the deployment closed before its turn (the chain converts nothing for + * those but still spends a slot on them, which shifts every later deployment's conversion block). + */ +export const ActMigrationQueue = akashSchema.table("act_migration_queue", { + position: integer("position").primaryKey(), + deploymentId: bigint("deployment_id", { mode: "number" }) + .notNull() + .references(() => Deployments.id), + convertedAtHeight: bigint("converted_at_height", { mode: "number" }) +}); + +/** + * Singleton tracker for the AKT/USD oracle price the drain uses: the chain's deployment EndBlocker + * reads the aggregated price stored at the previous block's oracle EndBlocker, so each drain block + * converts at the latest `EventPriceData` from strictly earlier blocks. Prices are not otherwise + * persisted, and a restart mid-drain must not lose the last one seen. + */ +export const ActMigrationState = akashSchema.table( + "act_migration_state", + { + id: integer("id").primaryKey(), + lastAktUsdPrice: numeric("last_akt_usd_price", { precision: 38, scale: 18 }), + lastPriceHeight: bigint("last_price_height", { mode: "number" }) + }, + t => [check("act_migration_state_singleton_check", sql`${t.id} = 1`)] +); + /** * Singleton current network state, maintained incrementally inside the per-block transaction so * dashboards read one row instead of scanning leases. `last_aggregated_height` is the replay diff --git a/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts b/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts index 34f68a2b3c..d9717340d6 100644 --- a/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts +++ b/apps/chain-indexer/src/pipeline/block-committer.service.spec.ts @@ -6,6 +6,7 @@ import { mock } from "vitest-mock-extended"; import type { AkashWriter } from "@src/akash/akash-writer.service"; import type { NetworkBlockDelta } from "@src/akash/network-delta"; import type { ProviderWriter } from "@src/akash/provider-writer.service"; +import type { ActMigrationSegment, ActMigrationService } from "@src/bme/act-migration.service"; import type { BmeWriter } from "@src/bme/bme-writer.service"; import { AccountTxs, Blocks, IndexerState, MessageDeadLetters, Messages, MessageTypes } from "@src/db/schema"; import type { GovWriter } from "@src/gov/gov-writer.service"; @@ -19,6 +20,7 @@ import type { LoggerService } from "@src/providers/logging.provider"; const MSG_SEND = "/cosmos.bank.v1beta1.MsgSend"; const BALANCE_WRITE = Symbol("balance_write"); +const ACT_MIGRATION_APPLY = Symbol("act_migration_apply"); describe(BlockCommitterService.name, () => { it("reuses ids of message types that already exist instead of inserting them", async () => { @@ -110,6 +112,40 @@ describe(BlockCommitterService.name, () => { }); }); + describe("ACT migration", () => { + it("commits each migration segment separately and applies the segment's step before its checkpoint", async () => { + const { committer, actMigration, insertedRows } = setup({ selectResults: [[{ id: 7, type: MSG_SEND }]] }); + const blocks = [buildBlock([MSG_SEND], 10), buildBlock([MSG_SEND], 11), buildBlock([MSG_SEND], 12)]; + const observations = { lastAktUsdPrice: null }; + const segments: ActMigrationSegment[] = [ + { + blocks: blocks.slice(0, 2), + step: { kind: "upgrade", height: 11, bankTotals: { burnedUakt: 0n, burnedUsdc: 0n, mintedUact: 0n }, validatable: true }, + observations + }, + { blocks: blocks.slice(2), step: null, observations } + ]; + actMigration.segment.mockResolvedValue(segments); + const outcome = { upgradeApplied: true, queueRemaining: 3, drained: false }; + actMigration.applySegment.mockImplementation(async (_tx, segment) => { + insertedRows.push({ table: ACT_MIGRATION_APPLY, rows: [] }); + return segment.step ? outcome : null; + }); + + await committer.commitBatch(blocks, { stream: "backfill:10-12" }); + + const checkpointRows = insertedRows.filter(call => call.table === IndexerState); + expect(checkpointRows.map(call => call.rows)).toEqual([expect.objectContaining({ lastHeight: 11 }), expect.objectContaining({ lastHeight: 12 })]); + const applyIndex = insertedRows.findIndex(call => call.table === ACT_MIGRATION_APPLY); + const firstCheckpointIndex = insertedRows.findIndex(call => call.table === IndexerState); + expect(applyIndex).toBeGreaterThan(-1); + expect(applyIndex).toBeLessThan(firstCheckpointIndex); + expect(actMigration.applySegment).toHaveBeenCalledTimes(2); + expect(actMigration.markCommitted).toHaveBeenNthCalledWith(1, segments[0], outcome); + expect(actMigration.markCommitted).toHaveBeenNthCalledWith(2, segments[1], null); + }); + }); + describe("dead letters", () => { const UNKNOWN_TYPE = "/akash.unknown.v1.MsgMystery"; @@ -367,6 +403,9 @@ describe(BlockCommitterService.name, () => { const providerWriter = mock(); const bmeWriter = mock(); const networkStatsWriter = mock(); + const actMigration = mock(); + actMigration.segment.mockImplementation(async blocks => [{ blocks, step: null, observations: { lastAktUsdPrice: null } }]); + actMigration.applySegment.mockResolvedValue(null); const logger = mock(); const committer = new BlockCommitterService( dbFake as unknown as ChainDatabase, @@ -377,6 +416,7 @@ describe(BlockCommitterService.name, () => { providerWriter, bmeWriter, networkStatsWriter, + actMigration, logger ); return { @@ -391,6 +431,7 @@ describe(BlockCommitterService.name, () => { providerWriter, bmeWriter, networkStatsWriter, + actMigration, logger }; } diff --git a/apps/chain-indexer/src/pipeline/block-committer.service.ts b/apps/chain-indexer/src/pipeline/block-committer.service.ts index da9a856684..8c3d91144c 100644 --- a/apps/chain-indexer/src/pipeline/block-committer.service.ts +++ b/apps/chain-indexer/src/pipeline/block-committer.service.ts @@ -7,6 +7,8 @@ import { collectAkashAddresses } from "@src/akash/akash-changes"; import { deriveAkashChanges } from "@src/akash/akash-deriver"; import { AkashWriter } from "@src/akash/akash-writer.service"; import { ProviderWriter } from "@src/akash/provider-writer.service"; +import type { ActMigrationOutcome, ActMigrationSegment } from "@src/bme/act-migration.service"; +import { ActMigrationService } from "@src/bme/act-migration.service"; import type { BmeBlockChanges } from "@src/bme/bme-deriver"; import { collectBmeAddresses, deriveBmeChanges } from "@src/bme/bme-deriver"; import { BmeWriter } from "@src/bme/bme-writer.service"; @@ -55,6 +57,7 @@ export class BlockCommitterService { readonly #providerWriter: ProviderWriter; readonly #bmeWriter: BmeWriter; readonly #networkStatsWriter: NetworkStatsWriter; + readonly #actMigration: ActMigrationService; readonly #logger: LoggerService; readonly #moduleRegistry = buildModuleAddressRegistry(); readonly #typeIds = new Map(); @@ -68,6 +71,7 @@ export class BlockCommitterService { @inject(ProviderWriter) providerWriter: ProviderWriter, @inject(BmeWriter) bmeWriter: BmeWriter, @inject(NetworkStatsWriter) networkStatsWriter: NetworkStatsWriter, + @inject(ActMigrationService) actMigration: ActMigrationService, @inject(LoggerService) logger: LoggerService ) { this.#db = db; @@ -78,6 +82,7 @@ export class BlockCommitterService { this.#providerWriter = providerWriter; this.#bmeWriter = bmeWriter; this.#networkStatsWriter = networkStatsWriter; + this.#actMigration = actMigration; this.#logger = logger; this.#logger.setContext("COMMITTER"); } @@ -91,6 +96,9 @@ export class BlockCommitterService { * never points past uncommitted data. Inserts are conflict-ignoring and the checkpoint only * moves forward, so concurrent writers on the same stream (e.g. two pods overlapping during a * rolling deploy) duplicate work but cannot corrupt data or regress the checkpoint. + * + * A pending ACT denom migration splits the batch at its trigger block: the conversion commits in + * the same transaction as that block, so later blocks' settlements run against converted state. */ async commitBatch(blocks: DecodedBlock[], options: { stream: string }): Promise { if (blocks.length === 0) { @@ -98,6 +106,14 @@ export class BlockCommitterService { } this.#verifyContiguous(blocks); + + for (const segment of await this.#actMigration.segment(blocks)) { + const outcome = await this.#commitSegment(segment.blocks, options, segment); + this.#actMigration.markCommitted(segment, outcome); + } + } + + async #commitSegment(blocks: DecodedBlock[], options: { stream: string }, segment: ActMigrationSegment): Promise { const typeIds = await this.#internMessageTypes(blocks); const blockRows = blocks.map(block => ({ @@ -153,7 +169,7 @@ export class BlockCommitterService { const lastHeight = blocks[blocks.length - 1].height; - const persistedDeadLetters = await this.#db.transaction(async tx => { + const { persistedDeadLetters, migrationOutcome } = await this.#db.transaction(async tx => { await insertChunked(tx, Blocks, blockRows); await insertChunked(tx, Transactions, transactionRows); await this.#upsertMessages(tx, messageRows); @@ -167,6 +183,8 @@ export class BlockCommitterService { await this.#bmeWriter.write(tx, bmeChanges, accountIds); await this.#networkStatsWriter.write(tx, blocks, networkDeltas); + const outcome = await this.#actMigration.applySegment(tx, segment); + await tx .insert(IndexerState) .values({ stream: options.stream, lastHeight, updatedAt: new Date() }) @@ -175,7 +193,7 @@ export class BlockCommitterService { set: { lastHeight: sql`GREATEST(${IndexerState.lastHeight}, EXCLUDED.last_height)`, updatedAt: new Date() } }); - return persisted; + return { persistedDeadLetters: persisted, migrationOutcome: outcome }; }); if (persistedDeadLetters.length > 0) { @@ -188,6 +206,8 @@ export class BlockCommitterService { toHeight: lastHeight }); } + + return migrationOutcome; } /** diff --git a/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts b/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts index 0437e2d278..0e9a2bdf1e 100644 --- a/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts +++ b/apps/chain-indexer/src/pipeline/block-decoder.service.spec.ts @@ -234,7 +234,7 @@ describe(BlockDecoderService.name, () => { expect(decoded.blockEvents).toEqual([{ type: "coinbase", attributes: { minter: "akash1mint", amount: "10uakt" } }]); }); - it("captures bme lifecycle block events for the bme handler but drops vault funded ones", () => { + it("captures every native bme block event, including the vault funding that marks the upgrade block", () => { const { decoder } = setup(); const decoded = decoder.decode( @@ -244,7 +244,8 @@ describe(BlockDecoderService.name, () => { event("akash.bme.v1.EventLedgerRecordExecuted", { id: '{"denom":"uakt","to_denom":"uact","source":"bme","height":100,"sequence":1}' }), event("akash.bme.v1.EventMintStatusChange", { previous_status: '"mint_status_healthy"', new_status: '"mint_status_warning"' }), event("akash.bme.v1.EventLedgerRecordCanceled", { cancel_reason: '"epsilon"' }), - event("akash.bme.v1.EventVaultFunded", { source: '"bme"' }) + event("akash.bme.v1.EventVaultFunded", { source: '"bme"' }), + event("akash.provider.v1beta4.EventProviderCreated", { owner: '"akash1p"' }) ] }) ); @@ -252,10 +253,24 @@ describe(BlockDecoderService.name, () => { expect(decoded.blockEvents.map(e => e.type)).toEqual([ "akash.bme.v1.EventLedgerRecordExecuted", "akash.bme.v1.EventMintStatusChange", - "akash.bme.v1.EventLedgerRecordCanceled" + "akash.bme.v1.EventLedgerRecordCanceled", + "akash.bme.v1.EventVaultFunded" ]); }); + it("captures oracle price events so the ACT migration can track the conversion rate", () => { + const { decoder } = setup(); + + const decoded = decoder.decode( + buildBlock({ txs: [] }), + buildBlockResults([], { + finalize_block_events: [event("akash.oracle.v1.EventPriceData", { id: '{"denom":"akt","base_denom":"usd"}', data: '{"price":"0.62","timestamp":"t"}' })] + }) + ); + + expect(decoded.blockEvents.map(e => e.type)).toEqual(["akash.oracle.v1.EventPriceData"]); + }); + it("falls back to begin and end block events when finalize is absent", () => { const { decoder } = setup(); diff --git a/apps/chain-indexer/src/pipeline/block-decoder.service.ts b/apps/chain-indexer/src/pipeline/block-decoder.service.ts index 2561ee8370..173ab76e1e 100644 --- a/apps/chain-indexer/src/pipeline/block-decoder.service.ts +++ b/apps/chain-indexer/src/pipeline/block-decoder.service.ts @@ -20,9 +20,11 @@ import type { RpcBlockResult, RpcBlockResultsResult, RpcEvent, RpcTxResult } fro * proposal ids and lifecycle transitions for the governance handler; the akash events catch deployment and * lease closes that happen as side effects of other messages (group close, overdraw) — `akash.v1` is the * legacy string-attribute event of early mainnet, the typed events are the current chain's. The bme events - * carry the executed/canceled ledger records and mint status transitions the BME handler derives from - * (EventVaultFunded is deliberately absent: the vault's balance is already exact in the balance ledger). - * Capturing the rest would waste memory across backfill batches. + * carry the executed/canceled ledger records and mint status transitions the BME handler derives from; + * EventVaultFunded contributes nothing to the balance ledger (the vault's balance is already exact there) + * but it is the only native BME event the upgrade block emits, so the ACT denom migration needs it to + * detect the upgrade. The oracle price events supply that migration's conversion rate. Capturing the rest + * would waste memory across backfill batches. */ const RELEVANT_EVENT_TYPES = new Set([ "coin_spent", @@ -39,7 +41,9 @@ const RELEVANT_EVENT_TYPES = new Set([ "akash.market.v1.EventLeaseClosed", "akash.bme.v1.EventLedgerRecordExecuted", "akash.bme.v1.EventMintStatusChange", - "akash.bme.v1.EventLedgerRecordCanceled" + "akash.bme.v1.EventLedgerRecordCanceled", + "akash.bme.v1.EventVaultFunded", + "akash.oracle.v1.EventPriceData" ]); const MSG_INDEX_ATTRIBUTE = "msg_index";