Add incremental backup support

src/backup/backupCreator.ts

@@ -14,6 +14,7 @@ import {
   validateBackupId,
   validateExcludeClassNames,
   validateIncludeClassNames,
+  validateIncrementalBaseBackupId,
 } from './validation.js';
 
 const WAIT_INTERVAL = 1000;
@@ -26,6 +27,7 @@ export default class BackupCreator extends CommandBase {
   private statusGetter: BackupCreateStatusGetter;
   private waitForCompletion!: boolean;
   private config?: BackupConfig;
+  private incrementalBaseBackupId?: string;
 
   constructor(client: Connection, statusGetter: BackupCreateStatusGetter) {
     super(client);
@@ -70,12 +72,27 @@ export default class BackupCreator extends CommandBase {
     return this;
   }
 
+  /**
+   * The ID of an existing backup to use as the base for a file-based incremental backup.
+   * Only the files that changed since the base backup are included in the new backup.
+   *
+   * `backupId` is a plain backup ID string: either a literal, e.g. `'my-base-backup'`, or the
+   * `id` returned by a previous backup creation.
+   *
+   * Requires Weaviate v1.37.0 or higher.
+   */
+  withIncrementalBaseBackupId(backupId: string) {
+    this.incrementalBaseBackupId = backupId;
+    return this;
+  }
+
   validate = (): void => {
     this.addErrors([
       ...validateIncludeClassNames(this.includeClassNames),
       ...validateExcludeClassNames(this.excludeClassNames),
       ...validateBackend(this.backend),
       ...validateBackupId(this.backupId),
+      ...validateIncrementalBaseBackupId(this.incrementalBaseBackupId, this.backupId),
     ]);
   };
 
@@ -90,6 +107,7 @@ export default class BackupCreator extends CommandBase {
       config: this.config,
       include: this.includeClassNames,
       exclude: this.excludeClassNames,
+      incremental_base_backup_id: this.incrementalBaseBackupId,
     } as BackupCreateRequest;
 
     if (this.waitForCompletion) {

src/backup/validation.ts

@@ -45,3 +45,18 @@ export function validateBackupId(backupId?: string) {
   }
   return [];
 }
+
+export function validateIncrementalBaseBackupId(incrementalBaseBackupId?: string, backupId?: string) {
+  if (incrementalBaseBackupId === undefined || incrementalBaseBackupId === null) {
+    return [];
+  }
+  if (!isValidStringProperty(incrementalBaseBackupId)) {
+    return [
+      'string incrementalBaseBackupId must be a non-empty string - set with .withIncrementalBaseBackupId(backupId)',
+    ];
+  }
+  if (incrementalBaseBackupId === backupId) {
+    return ['incrementalBaseBackupId must be different from the ID of the backup being created'];
+  }
+  return [];
+}

src/collections/backup/client.ts

@@ -5,7 +5,11 @@ import {
   BackupRestoreStatusGetter,
   BackupRestorer,
 } from '../../backup/index.js';
-import { validateBackend, validateBackupId } from '../../backup/validation.js';
+import {
+  validateBackend,
+  validateBackupId,
+  validateIncrementalBaseBackupId,
+} from '../../backup/validation.js';
 import Connection from '../../connection/index.js';
 import {
   WeaviateBackupCanceled,
@@ -14,24 +18,27 @@ import {
   WeaviateInvalidInputError,
   WeaviateUnexpectedResponseError,
   WeaviateUnexpectedStatusCodeError,
+  WeaviateUnsupportedFeatureError,
 } from '../../errors.js';
 import {
   BackupCreateResponse,
   BackupCreateStatusResponse,
+  BackupListResponse,
   BackupRestoreResponse,
 } from '../../openapi/types.js';
+import { DbVersionSupport } from '../../utils/dbVersion.js';
 import {
   BackupArgs,
   BackupCancelArgs,
-  BackupConfigCreate,
   BackupConfigRestore,
+  BackupCreateArgs,
   BackupReturn,
   BackupStatusArgs,
   BackupStatusReturn,
   ListBackupOptions,
 } from './types.js';
 
-export const backup = (connection: Connection): Backup => {
+export const backup = (connection: Connection, dbVersionSupport: DbVersionSupport): Backup => {
   const parseStatus = (res: BackupCreateStatusResponse | BackupRestoreResponse): BackupStatusReturn => {
     if (res.id === undefined) {
       throw new WeaviateUnexpectedResponseError('Backup ID is undefined in response');
@@ -47,6 +54,9 @@ export const backup = (connection: Connection): Backup => {
       error: res.error,
       path: res.path,
       status: res.status,
+      // Only returned by Weaviate >=1.37, for incremental backups, and only to root users
+      incrementalBaseBackupId:
+        'incremental_base_backup_id' in res ? res.incremental_base_backup_id || undefined : undefined,
     };
   };
   const parseResponse = (res: BackupCreateResponse | BackupRestoreResponse): BackupReturn => {
@@ -109,10 +119,22 @@ export const backup = (connection: Connection): Backup => {
 
       return true;
     },
-    create: async (args: BackupArgs<BackupConfigCreate>): Promise<BackupReturn> => {
+    create: async (args: BackupCreateArgs): Promise<BackupReturn> => {
       let builder = new BackupCreator(connection, new BackupCreateStatusGetter(connection))
         .withBackupId(args.backupId)
         .withBackend(args.backend);
+      if (args.incrementalBaseBackupId !== undefined) {
+        const baseBackupId = args.incrementalBaseBackupId.toLowerCase();
+        const errors = validateIncrementalBaseBackupId(baseBackupId, args.backupId.toLowerCase());
+        if (errors.length > 0) {
+          throw new WeaviateInvalidInputError(errors.join(', '));
+        }
+        const check = await dbVersionSupport.supportsIncrementalBackups();
+        if (!check.supports) {
+          throw new WeaviateUnsupportedFeatureError(check.message);
+        }
+        builder = builder.withIncrementalBaseBackupId(baseBackupId);
+      }
       if (args.includeCollections) {
         builder = builder.withIncludeClassNames(...args.includeCollections);
       }
@@ -213,7 +235,12 @@ export const backup = (connection: Connection): Backup => {
       if (opts?.startedAtAsc) {
         url += '?order=asc';
       }
-      return connection.get<BackupReturn[]>(url);
+      return connection.get<BackupListResponse>(url).then((res) =>
+        res.map(({ incremental_base_backup_id: baseBackupId, ...rest }) => ({
+          ...rest,
+          incrementalBaseBackupId: baseBackupId || undefined,
+        }))
+      ) as Promise<BackupReturn[]>;
     },
   };
 };
@@ -231,13 +258,17 @@ export interface Backup {
   /**
    * Create a backup of the database.
    *
-   * @param {BackupArgs} args The arguments for the request.
+   * Pass `incrementalBaseBackupId` to create a file-based incremental backup, which only
+   * contains the files that changed since the given base backup. Requires Weaviate `v1.37.0` or higher.
+   *
+   * @param {BackupCreateArgs} args The arguments for the request.
    * @returns {Promise<BackupReturn>} The response from Weaviate.
    * @throws {WeaviateInvalidInputError} If the input is invalid.
+   * @throws {WeaviateUnsupportedFeatureError} If `incrementalBaseBackupId` is used with Weaviate <1.37.0.
    * @throws {WeaviateBackupFailed} If the backup creation fails.
    * @throws {WeaviateBackupCanceled} If the backup creation is canceled.
    */
-  create(args: BackupArgs<BackupConfigCreate>): Promise<BackupReturn>;
+  create(args: BackupCreateArgs): Promise<BackupReturn>;
   /**
    * Get the status of a backup creation.
    *

src/collections/backup/collection.ts

@@ -1,5 +1,6 @@
 import { Backend } from '../../backup/index.js';
 import Connection from '../../connection/index.js';
+import { DbVersionSupport } from '../../utils/dbVersion.js';
 import { backup } from './client.js';
 import { BackupReturn, BackupStatusArgs, BackupStatusReturn } from './types.js';
 
@@ -13,10 +14,28 @@ export type BackupCollectionArgs = {
   waitForCompletion?: boolean;
 };
 
-export const backupCollection = (connection: Connection, name: string) => {
-  const handler = backup(connection);
+/** The arguments required to create a backup of a collection. */
+export type BackupCollectionCreateArgs = BackupCollectionArgs & {
+  /**
+   * The ID of an existing backup to use as the base for a file-based incremental backup.
+   * If set, only the files that have changed since the base backup are included in the new backup.
+   *
+   * This is a plain backup ID string: either a literal, e.g. `'my-base-backup'`, or the `id`
+   * returned by a previous backup creation.
+   *
+   * Requires Weaviate `v1.37.0` or higher.
+   */
+  incrementalBaseBackupId?: string;
+};
+
+export const backupCollection = (
+  connection: Connection,
+  name: string,
+  dbVersionSupport: DbVersionSupport
+) => {
+  const handler = backup(connection, dbVersionSupport);
   return {
-    create: (args: BackupCollectionArgs) =>
+    create: (args: BackupCollectionCreateArgs) =>
       handler.create({
         ...args,
         includeCollections: [name],
@@ -35,13 +54,17 @@ export interface BackupCollection {
   /**
    * Create a backup of this collection.
    *
-   * @param {BackupArgs} args The arguments for the request.
+   * Pass `incrementalBaseBackupId` to create a file-based incremental backup, which only
+   * contains the files that changed since the given base backup. Requires Weaviate `v1.37.0` or higher.
+   *
+   * @param {BackupCollectionCreateArgs} args The arguments for the request.
    * @returns {Promise<BackupReturn>} The response from Weaviate.
    * @throws {WeaviateInvalidInputError} If the input is invalid.
+   * @throws {WeaviateUnsupportedFeatureError} If `incrementalBaseBackupId` is used with Weaviate <1.37.0.
    * @throws {WeaviateBackupFailed} If the backup creation fails.
    * @throws {WeaviateBackupCanceled} If the backup creation is canceled.
    */
-  create(args: BackupCollectionArgs): Promise<BackupReturn>;
+  create(args: BackupCollectionCreateArgs): Promise<BackupReturn>;
   /**
    * Get the status of a backup.
    *

src/collections/backup/index.ts

@@ -1,3 +1,9 @@
 export type { Backup } from './client.js';
-export type { BackupCollection, BackupCollectionArgs } from './collection.js';
-export type { BackupArgs, BackupConfigCreate, BackupConfigRestore, BackupStatusArgs } from './types.js';
+export type { BackupCollection, BackupCollectionArgs, BackupCollectionCreateArgs } from './collection.js';
+export type {
+  BackupArgs,
+  BackupConfigCreate,
+  BackupConfigRestore,
+  BackupCreateArgs,
+  BackupStatusArgs,
+} from './types.js';

src/collections/backup/types.ts

@@ -16,6 +16,8 @@ export type BackupStatusReturn = {
   status: BackupStatus;
   /** Size of the backup in Gibs */
   size?: number;
+  /** The ID of the base backup this incremental backup was built on; undefined if the backup is not incremental. */
+  incrementalBaseBackupId?: string;
 };
 
 /** The return type of a backup creation or restoration operation */
@@ -64,6 +66,20 @@ export type BackupArgs<C extends BackupConfigCreate | BackupConfigRestore> = {
   config?: C;
 };
 
+/** The arguments required to create a backup. */
+export type BackupCreateArgs = BackupArgs<BackupConfigCreate> & {
+  /**
+   * The ID of an existing backup to use as the base for a file-based incremental backup.
+   * If set, only the files that have changed since the base backup are included in the new backup.
+   *
+   * This is a plain backup ID string: either a literal, e.g. `'my-base-backup'`, or the `id`
+   * returned by a previous backup creation.
+   *
+   * Requires Weaviate `v1.37.0` or higher.
+   */
+  incrementalBaseBackupId?: string;
+};
+
 /** The arguments required to get the status of a backup. */
 export type BackupStatusArgs = {
   /** The ID of the backup. */

src/collections/collection/index.ts

@@ -138,7 +138,7 @@ const collection = <T, N, V>(
   );
   return {
     aggregate: aggregateCollection,
-    backup: backupCollection(connection, capitalizedName),
+    backup: backupCollection(connection, capitalizedName, dbVersionSupport),
     config: config<T>(connection, capitalizedName, dbVersionSupport, tenant),
     data: data<T>(connection, capitalizedName, dbVersionSupport, consistencyLevel, tenant),
     filter: filter<T extends undefined ? any : T>(),

src/index.ts

@@ -233,7 +233,7 @@ async function client(params: ClientParams): Promise<WeaviateClient> {
 
   const ifc: WeaviateClient = {
     alias: alias(connection),
-    backup: backup(connection),
+    backup: backup(connection, dbVersionSupport),
     batch: batch(connection, dbVersionSupport),
     cluster: cluster(connection),
     collections: collections(connection, dbVersionSupport),

src/openapi/types.ts

@@ -15,6 +15,7 @@ export type DataObject = definitions['Object'];
 export type BackupCreateRequest = definitions['BackupCreateRequest'];
 export type BackupCreateResponse = definitions['BackupCreateResponse'];
 export type BackupCreateStatusResponse = definitions['BackupCreateStatusResponse'];
+export type BackupListResponse = definitions['BackupListResponse'];
 export type BackupRestoreRequest = definitions['BackupRestoreRequest'];
 export type BackupRestoreResponse = definitions['BackupRestoreResponse'];
 export type BackupRestoreStatusResponse = definitions['BackupRestoreStatusResponse'];

src/utils/dbVersion.ts

@@ -163,6 +163,13 @@ export class DbVersionSupport {
       message: this.errorMessage('Tokenize endpoint stopwords / stopwordPresets', version.show(), '1.37.2'),
     }));
 
+  supportsIncrementalBackups = () =>
+    this.dbVersionProvider.getVersion().then((version) => ({
+      version,
+      supports: version.isAtLeast(1, 37, 0),
+      message: this.errorMessage('Incremental backups', version.show(), '1.37.0'),
+    }));
+
   supportsServerSideDefaultVectorIndexType = () =>
     this.dbVersionProvider.getVersion().then((version) => ({
       version,

test/collections/backup/integration.test.ts

@@ -2,7 +2,7 @@
 /* eslint-disable @typescript-eslint/no-non-null-asserted-optional-chain */
 /* eslint-disable no-await-in-loop */
 import { afterAll, beforeAll, describe, expect, it, test } from 'vitest';
-import { WeaviateBackupFailed } from '../../../src/errors.js';
+import { WeaviateBackupFailed, WeaviateInvalidInputError } from '../../../src/errors.js';
 import weaviate, { Backend, Collection, WeaviateClient } from '../../../src/index.js';
 import { requireAtLeast } from '../../../test/version.js';
 
@@ -258,6 +258,65 @@ describe('Integration testing of backups', () => {
     });
   });
 
+  requireAtLeast(1, 37, 0).describe('incremental backups', () => {
+    it('creates an incremental backup on top of a base backup', async () => {
+      const client = await clientPromise;
+      const collection = await client.collections
+        .create({ name: 'TestIncrementalBackup' })
+        .then((col) => col.data.insert().then(() => col));
+
+      const base = await client.backup.create({
+        backupId: randomBackupId(),
+        backend: 'filesystem',
+        includeCollections: [collection.name],
+        waitForCompletion: true,
+      });
+      expect(base.status).toBe('SUCCESS');
+
+      // Add data so that the incremental backup has something to pick up
+      await collection.data.insert();
+
+      const incremental = await client.backup.create({
+        backupId: randomBackupId(),
+        backend: 'filesystem',
+        includeCollections: [collection.name],
+        incrementalBaseBackupId: base.id,
+        waitForCompletion: true,
+      });
+      expect(incremental.status).toBe('SUCCESS');
+
+      // Weaviate only reports the base backup ID to root users, so treat it as optional
+      if (incremental.incrementalBaseBackupId !== undefined) {
+        expect(incremental.incrementalBaseBackupId).toBe(base.id);
+      }
+
+      // The incremental backup must be restorable
+      await client.collections.delete(collection.name);
+      const restored = await client.backup.restore({
+        backupId: incremental.id,
+        backend: 'filesystem',
+        includeCollections: [collection.name],
+        waitForCompletion: true,
+      });
+      expect(restored.status).toBe('SUCCESS');
+      await expect(collection.length()).resolves.toBe(2);
+
+      await client.collections.delete(collection.name);
+    });
+
+    it('rejects an incremental backup based on itself', async () => {
+      const client = await clientPromise;
+      const backupId = randomBackupId();
+      await expect(
+        client.backup.create({
+          backupId,
+          backend: 'filesystem',
+          incrementalBaseBackupId: backupId,
+        })
+      ).rejects.toThrow(WeaviateInvalidInputError);
+    });
+  });
+
   function randomBackupId() {
     return 'backup-id-' + Math.floor(Math.random() * Number.MAX_SAFE_INTEGER);
   }