Use an index to find rows in the next page in the AggsUTXO query

The indexer was using SQL's `skip` for the pagination. The DB should
scan the number of skipped rows, to get the page result. For the
performance enhancement, we concluded that change the pagination
method. Finding a particular row using an index and get the following
`n` rows is much faster than using `skip`.

This commit uses `firstEvaluatedKey` and `lastEvaluatedKey` for the
pagination in the AggsUTXO query. The AggsUTXO query will find a row
using `firstEvaluatedKey` or `lastEvaluatedKey` and returns next `n`
rows.

Author: Park Juhyung

src/models/logic/aggsUTXO.ts

@@ -2,17 +2,30 @@ import { H160 } from "codechain-primitives";
 import * as Sequelize from "sequelize";
 import models from "..";
 import * as Exception from "../../exception";
+import { aggsUTXOPagination } from "../../routers/pagination";
 
 export async function getByAddress(params: {
     address: string;
     assetType?: H160 | null;
     page?: number | null;
     itemsPerPage?: number | null;
+    firstEvaluatedKey?: [number] | null;
+    lastEvaluatedKey?: [number] | null;
 }) {
-    const { address, assetType, page = 1, itemsPerPage = 15 } = params;
+    const {
+        address,
+        assetType,
+        page = 1,
+        itemsPerPage = 15,
+        firstEvaluatedKey,
+        lastEvaluatedKey
+    } = params;
     const query: any = getAggsUTXOQuery({
         address,
-        assetType
+        assetType,
+        order: "assetType",
+        firstEvaluatedKey,
+        lastEvaluatedKey
     });
 
     const includeArray: any = [
@@ -27,9 +40,15 @@ export async function getByAddress(params: {
             where: {
                 [Sequelize.Op.and]: query
             },
-            order: [["address", "DESC"], ["assetType", "DESC"]],
+            order: aggsUTXOPagination.byAssetType.orderby({
+                firstEvaluatedKey,
+                lastEvaluatedKey
+            }),
             limit: itemsPerPage!,
-            offset: (page! - 1) * itemsPerPage!,
+            offset:
+                firstEvaluatedKey || lastEvaluatedKey
+                    ? 0
+                    : (page! - 1) * itemsPerPage!,
             include: includeArray
         });
     } catch (err) {
@@ -43,11 +62,23 @@ export async function getByAssetType(params: {
     address?: string | null;
     page?: number | null;
     itemsPerPage?: number | null;
+    firstEvaluatedKey?: [number] | null;
+    lastEvaluatedKey?: [number] | null;
 }) {
-    const { address, assetType, page = 1, itemsPerPage = 15 } = params;
+    const {
+        address,
+        assetType,
+        page = 1,
+        itemsPerPage = 15,
+        firstEvaluatedKey,
+        lastEvaluatedKey
+    } = params;
     const query: any = getAggsUTXOQuery({
         address,
-        assetType
+        assetType,
+        order: "address",
+        firstEvaluatedKey,
+        lastEvaluatedKey
     });
 
     const includeArray: any = [
@@ -62,9 +93,15 @@ export async function getByAssetType(params: {
             where: {
                 [Sequelize.Op.and]: query
             },
-            order: [["assetType", "DESC"], ["address", "DESC"]],
+            order: aggsUTXOPagination.byAddress.orderby({
+                firstEvaluatedKey,
+                lastEvaluatedKey
+            }),
             limit: itemsPerPage!,
-            offset: (page! - 1) * itemsPerPage!,
+            offset:
+                firstEvaluatedKey || lastEvaluatedKey
+                    ? 0
+                    : (page! - 1) * itemsPerPage!,
             include: includeArray
         });
     } catch (err) {
@@ -76,8 +113,17 @@ export async function getByAssetType(params: {
 function getAggsUTXOQuery(params: {
     address?: string | null;
     assetType?: H160 | null;
+    order: "assetType" | "address";
+    firstEvaluatedKey?: [number] | null;
+    lastEvaluatedKey?: [number] | null;
 }) {
-    const { address, assetType } = params;
+    const {
+        order,
+        address,
+        assetType,
+        firstEvaluatedKey,
+        lastEvaluatedKey
+    } = params;
     const query = [];
     if (address) {
         query.push({
@@ -90,5 +136,23 @@ function getAggsUTXOQuery(params: {
         });
     }
 
+    if (firstEvaluatedKey || lastEvaluatedKey) {
+        if (order === "assetType") {
+            query.push(
+                aggsUTXOPagination.byAssetType.where({
+                    firstEvaluatedKey,
+                    lastEvaluatedKey
+                })
+            );
+        } else if (order === "address") {
+            query.push(
+                aggsUTXOPagination.byAddress.where({
+                    firstEvaluatedKey,
+                    lastEvaluatedKey
+                })
+            );
+        }
+    }
+
     return query;
 }

src/models/logic/utxo.ts

@@ -3,6 +3,7 @@ import * as Sequelize from "sequelize";
 import models from "..";
 import * as Exception from "../../exception";
 import { utxoPagination } from "../../routers/pagination";
+import { AggsUTXOAttribute } from "../aggsUTXO";
 import { AssetSchemeAttribute } from "../assetscheme";
 import { TransactionInstance } from "../transaction";
 import { AssetTransferOutput } from "../transferAsset";
@@ -309,31 +310,55 @@ export function createUTXOEvaluatedKey(utxo: UTXOAttribute): string {
     ]);
 }
 
+export function createAggsUTXOEvaluatedKey(
+    aggs: AggsUTXOAttribute,
+    order: "assetType" | "address"
+): string {
+    return JSON.stringify([
+        order === "assetType" ? aggs.assetType : aggs.address
+    ]);
+}
+
 export async function getAggsUTXO(params: {
+    order: "assetType" | "address";
     address?: string | null;
     assetType?: H160 | null;
     shardId?: number | null;
     page?: number | null;
     itemsPerPage?: number | null;
+    firstEvaluatedKey?: [number] | null;
+    lastEvaluatedKey?: [number] | null;
     onlyConfirmed?: boolean | null;
     confirmThreshold?: number | null;
 }) {
-    const { address, assetType, page = 1, itemsPerPage = 15 } = params;
+    const {
+        order,
+        address,
+        assetType,
+        page = 1,
+        itemsPerPage = 15,
+        firstEvaluatedKey,
+        lastEvaluatedKey
+    } = params;
 
     try {
-        if (address) {
+        if (order === "assetType") {
             return await AggsUTXOModel.getByAddress({
-                address,
+                address: address!,
                 assetType,
                 page,
-                itemsPerPage
+                itemsPerPage,
+                firstEvaluatedKey,
+                lastEvaluatedKey
             });
-        } else if (assetType) {
+        } else if (order === "address") {
             return await AggsUTXOModel.getByAssetType({
                 address,
-                assetType,
+                assetType: assetType!,
                 page,
-                itemsPerPage
+                itemsPerPage,
+                firstEvaluatedKey,
+                lastEvaluatedKey
             });
         } else {
             throw new Error("address == aggs == null");

src/routers/asset.ts

@@ -13,6 +13,7 @@ import {
 import * as UTXOModel from "../models/logic/utxo";
 import { createPaginationResult } from "./pagination";
 import {
+    aggsUTXOPaginationSchema,
     assetTypeSchema,
     paginationSchema,
     snapshotSchema,
@@ -312,6 +313,16 @@ export function handle(context: IndexerContext, router: Router) {
      *         in: query
      *         required: false
      *         type: number
+     *       - name: firstEvaluatedKey
+     *         description: the evaulated key of the first item in the previous page. It will be used for the pagination
+     *         in: query
+     *         required: false
+     *         type: string
+     *       - name: lastEvaluatedKey
+     *         description: the evaulated key of the last item in the previous page. It will be used for the pagination
+     *         in: query
+     *         required: false
+     *         type: string
      *       - name: onlyConfirmed
      *         description: returns only confirmed component
      *         in: query
@@ -336,10 +347,12 @@ export function handle(context: IndexerContext, router: Router) {
      */
     router.get(
         "/aggs-utxo",
+        parseEvaluatedKey,
         validate({
             query: {
                 ...utxoSchema,
-                ...paginationSchema
+                ...paginationSchema,
+                ...aggsUTXOPaginationSchema
             }
         }),
         syncIfNeeded(context),
@@ -350,27 +363,53 @@ export function handle(context: IndexerContext, router: Router) {
                 req.query.shardId && parseInt(req.query.shardId, 10);
             const page = req.query.page && parseInt(req.query.page, 10);
             const itemsPerPage =
-                req.query.itemsPerPage && parseInt(req.query.itemsPerPage, 10);
+                (req.query.itemsPerPage &&
+                    parseInt(req.query.itemsPerPage, 10)) ||
+                15;
             const onlyConfirmed = req.query.onlyConfirmed;
             const confirmThreshold =
                 req.query.confirmThreshold &&
                 parseInt(req.query.confirmThreshold, 10);
+            const firstEvaluatedKey = req.query.firstEvaluatedKey;
+            const lastEvaluatedKey = req.query.lastEvaluatedKey;
             let assetType;
+            let order: "assetType" | "address";
+            if (address) {
+                order = "assetType";
+            } else {
+                order = "address";
+            }
+
             try {
                 if (assetTypeString) {
                     assetType = new H160(assetTypeString);
                 }
                 const aggsInst = await UTXOModel.getAggsUTXO({
+                    order,
                     address,
                     assetType,
                     shardId,
                     page,
-                    itemsPerPage,
+                    itemsPerPage: itemsPerPage + 1,
+                    firstEvaluatedKey,
+                    lastEvaluatedKey,
                     onlyConfirmed,
                     confirmThreshold
                 });
-                const utxo = aggsInst.map(inst => inst.get({ plain: true }));
-                res.json(utxo);
+                const aggs = aggsInst.map(inst => inst.get({ plain: true }));
+
+                res.json(
+                    createPaginationResult({
+                        query: {
+                            firstEvaluatedKey,
+                            lastEvaluatedKey
+                        },
+                        rows: aggs,
+                        getEvaluatedKey: row =>
+                            UTXOModel.createAggsUTXOEvaluatedKey(row, order),
+                        itemsPerPage
+                    })
+                );
             } catch (e) {
                 next(e);
             }

src/routers/pagination.ts

@@ -143,3 +143,80 @@ export const blockPagination = {
         }
     }
 };
+
+export const aggsUTXOPagination = {
+    byAssetType: {
+        forwardOrder: [["assetType", "DESC"]],
+        reverseOrder: [["assetType", "ASC"]],
+        orderby: (params: {
+            firstEvaluatedKey?: number[] | null;
+            lastEvaluatedKey?: number[] | null;
+        }) => {
+            const order = queryOrder(params);
+            if (order === "forward") {
+                return aggsUTXOPagination.byAssetType.forwardOrder;
+            } else if (order === "reverse") {
+                return aggsUTXOPagination.byAssetType.reverseOrder;
+            }
+        },
+        where: (params: {
+            firstEvaluatedKey?: number[] | null;
+            lastEvaluatedKey?: number[] | null;
+        }) => {
+            const order = queryOrder(params);
+            const { firstEvaluatedKey, lastEvaluatedKey } = params;
+            if (order === "forward") {
+                const assetType = lastEvaluatedKey![0];
+                return {
+                    assetType: {
+                        [Sequelize.Op.lt]: assetType
+                    }
+                };
+            } else if (order === "reverse") {
+                const assetType = firstEvaluatedKey![0];
+                return {
+                    assetType: {
+                        [Sequelize.Op.gt]: assetType
+                    }
+                };
+            }
+        }
+    },
+    byAddress: {
+        forwardOrder: [["address", "DESC"]],
+        reverseOrder: [["address", "ASC"]],
+        orderby: (params: {
+            firstEvaluatedKey?: number[] | null;
+            lastEvaluatedKey?: number[] | null;
+        }) => {
+            const order = queryOrder(params);
+            if (order === "forward") {
+                return aggsUTXOPagination.byAddress.forwardOrder;
+            } else if (order === "reverse") {
+                return aggsUTXOPagination.byAddress.reverseOrder;
+            }
+        },
+        where: (params: {
+            firstEvaluatedKey?: number[] | null;
+            lastEvaluatedKey?: number[] | null;
+        }) => {
+            const order = queryOrder(params);
+            const { firstEvaluatedKey, lastEvaluatedKey } = params;
+            if (order === "forward") {
+                const address = lastEvaluatedKey![0];
+                return {
+                    address: {
+                        [Sequelize.Op.lt]: address
+                    }
+                };
+            } else if (order === "reverse") {
+                const address = firstEvaluatedKey![0];
+                return {
+                    address: {
+                        [Sequelize.Op.gt]: address
+                    }
+                };
+            }
+        }
+    }
+};

src/routers/validator.ts

@@ -88,6 +88,11 @@ export const blockPaginationSchema = {
     firstEvaluatedKey: blockEvaluationKey
 };
 
+export const aggsUTXOPaginationSchema = {
+    firstEvaluatedKey: Joi.array().items(Joi.string()),
+    lastEvaluatedKey: Joi.array().items(Joi.string())
+};
+
 export const txSchema = {
     address,
     assetType: assetTypeSchema,