diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index d4fc5d35e..36fc6b3bd 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -6,6 +6,7 @@ import { v4 as uuid } from 'uuid'; import * as lib_mongo from '@powersync/lib-service-mongodb'; import { mongo } from '@powersync/lib-service-mongodb'; +import { CompatibilityContext } from '@powersync/service-sync-rules'; import { ObjectId } from 'bson'; import { generateSlotName } from '../utils/util.js'; import { BucketDefinitionMapping } from './implementation/BucketDefinitionMapping.js'; @@ -15,13 +16,14 @@ import { PowerSyncMongo } from './implementation/db.js'; import { getMongoStorageConfig, StorageConfig, SyncRuleDocumentBase } from './implementation/models.js'; import { MongoChecksumOptions } from './implementation/MongoChecksums.js'; import { - MongoPersistedSyncRulesContentV1, - MongoPersistedSyncRulesContentV3 + MongoPersistedReplicationStream, + MongoPersistedSyncConfigContentV1, + MongoPersistedSyncConfigContentV3 } from './implementation/MongoPersistedSyncRulesContent.js'; import { syncRuleStateUpdatePipeline } from './implementation/SyncRuleStateUpdate.js'; import { SyncRuleDocumentV1 } from './implementation/v1/models.js'; import { VersionedPowerSyncMongoV3 } from './implementation/v3/VersionedPowerSyncMongoV3.js'; -import { ReplicationStreamDocumentV3, SyncConfigDefinition } from './storage-index.js'; +import { ReplicationStreamDocumentV3, SyncConfigDefinition, SyncRuleConfigStateV3 } from './storage-index.js'; export interface MongoBucketStorageOptions { checksumOptions?: Omit; @@ -57,16 +59,23 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { // No-op } - getInstance(syncRules: storage.PersistedSyncRulesContent, options?: GetIntanceOptions): MongoSyncBucketStorage { - let { id, slot_name } = syncRules; + getInstance( + replicationStream: storage.PersistedReplicationStream, + options?: GetIntanceOptions + ): MongoSyncBucketStorage { + const syncRulesContent = + replicationStream instanceof MongoPersistedReplicationStream + ? replicationStream.toSyncConfigContent() + : replicationStream; + let { id, slot_name } = replicationStream; if ((typeof id as any) == 'bigint') { id = Number(id); } - const storageConfig = (syncRules as MongoPersistedSyncRulesContentV1).getStorageConfig(); - const storage = createMongoSyncBucketStorage( + const storageConfig = (syncRulesContent as MongoPersistedSyncConfigContentV1).getStorageConfig(); + const syncRuleStorage = createMongoSyncBucketStorage( this, id, - syncRules as MongoPersistedSyncRulesContentV1, + syncRulesContent as MongoPersistedSyncConfigContentV1, slot_name, undefined, { @@ -75,17 +84,17 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { } ); if (!options?.skipLifecycleHooks) { - this.iterateListeners((cb) => cb.syncStorageCreated?.(storage)); + this.iterateListeners((cb) => cb.syncStorageCreated?.(syncRuleStorage)); } - storage.registerListener({ + syncRuleStorage.registerListener({ batchStarted: (batch) => { batch.registerListener({ replicationEvent: (payload) => this.iterateListeners((cb) => cb.replicationEvent?.(payload)) }); } }); - return storage; + return syncRuleStorage; } async getSystemIdentifier(): Promise { @@ -106,8 +115,8 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { } async restartReplication(sync_rules_group_id: number) { - const next = await this.getNextSyncRulesContent(); - const active = await this.getActiveSyncRulesContent(); + const next = await this.getDeployingSyncConfigContent(); + const active = await this.getActiveSyncConfigContent(); if (next != null && next.id == sync_rules_group_id) { // We need to redo the "next" replication stream @@ -151,17 +160,87 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { } } + private isCompatible( + replicationStream: ReplicationStreamDocumentV3, + existingConfig: SyncConfigDefinition[], + options: storage.UpdateSyncRulesOptions + ): boolean { + if (options.config.plan == null) { + // Only support sync streams with serialized plans + return false; + } + + if (replicationStream.storage_version !== options.storageVersion) { + return false; + } + + if (existingConfig.length == 0) { + // Could technically be compatible, but there is no reason to re-use this stream. + return false; + } + + if (existingConfig.some((config) => config.serialized_plan == null)) { + // Only support sync streams with serialized plans + return false; + } + + // Technically we can compare the serialized compatibility versions? But this does not add much overhead. + const first = existingConfig[0]; + const streamCompatibility = CompatibilityContext.deserialize(first.serialized_plan!.compatibility); + if (!streamCompatibility.equals(options.config.parsed.config.compatibility)) { + // Compatibility options must match + return false; + } + + return true; + } + private async updateSyncRulesV3( options: storage.UpdateSyncRulesOptions, storageVersion: number, storageConfig: StorageConfig - ): Promise { - let rules: MongoPersistedSyncRulesContentV3 | undefined = undefined; + ): Promise { + let rules: MongoPersistedReplicationStream | undefined = undefined; const versioned = this.db.versioned(storageConfig) as VersionedPowerSyncMongoV3; const session = this.session; await session.withTransaction(async () => { + const active = await this.db.sync_rules.findOne( + { + state: storage.SyncRuleState.ACTIVE, + storage_version: storageVersion + }, + { session, sort: { _id: -1 }, limit: 1 } + ); + if (active != null) { + const activeConfigs = active.sync_configs.filter((config) => config.state == storage.SyncRuleState.ACTIVE); + const activeOnly = { ...active, sync_configs: activeConfigs }; + const existingConfigDocs = await this.loadSyncConfigDefinitions(versioned, activeOnly, session); + + if (this.isCompatible(activeOnly, existingConfigDocs, options)) { + await this.db.sync_rules.updateMany( + { + state: storage.SyncRuleState.PROCESSING + }, + syncRuleStateUpdatePipeline(storage.SyncRuleState.STOP), + { session } + ); + await this.stopEmbeddedDeployingConfigs(active, session); + rules = await this.appendSyncConfigToStream({ + versioned, + existing: activeOnly, + existingConfigDocs, + options, + storageVersion, + session + }); + return; + } + + await this.stopEmbeddedDeployingConfigs(active, session); + } + // Only have a single replication stream with PROCESSING. await this.db.sync_rules.updateMany( { @@ -190,7 +269,12 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { const id = Number(id_doc!.op_id); const slot_name = generateSlotName(this.slot_name_prefix, id); - const mapping = BucketDefinitionMapping.fromParsedSyncRules(options.config.parsed); + const mapping = + options.config.plan == null + ? // For legacy sync rules and streams, use the parsed config directly to create a mapping + BucketDefinitionMapping.fromParsedSyncConfig(options.config.parsed) + : // For new sync streams, always use the serialized version + BucketDefinitionMapping.constructIncrementalMappingFromSerializedPlans([], options.config.plan.plan); const syncConfigDoc: SyncConfigDefinition = { _id: new ObjectId(), @@ -228,7 +312,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { await this.db.sync_rules.insertOne(doc, { session }); await this.db.notifyCheckpoint(); - rules = new MongoPersistedSyncRulesContentV3(this.db, doc, syncConfigDoc); + rules = new MongoPersistedReplicationStream(this.db, doc, [syncConfigDoc]); if (options.lock) { // The lock is persisted on rules.current_lock await rules.lock(session); @@ -238,9 +322,117 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { return rules!; } - async updateSyncRules( - options: storage.UpdateSyncRulesOptions - ): Promise { + private async loadSyncConfigDefinitions( + versioned: VersionedPowerSyncMongoV3, + existing: ReplicationStreamDocumentV3, + session: mongo.ClientSession + ) { + const activeConfigIds = existing.sync_configs + .filter((config) => [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.PROCESSING].includes(config.state)) + .map((config) => config._id); + + return versioned.syncConfigDefinitions + .find( + { + _id: { $in: activeConfigIds } + }, + { session } + ) + .toArray(); + } + + private async stopEmbeddedDeployingConfigs(existing: ReplicationStreamDocumentV3, session: mongo.ClientSession) { + const deployingConfigs = existing.sync_configs + .filter((config) => config.state == storage.SyncRuleState.PROCESSING) + .map((config) => config._id); + if (deployingConfigs.length == 0) { + return; + } + + await this.db.sync_rules.updateOne( + { + _id: existing._id, + 'sync_configs._id': { $in: deployingConfigs } + }, + { + $set: { + 'sync_configs.$[config].state': storage.SyncRuleState.STOP + } + }, + { + session, + arrayFilters: [{ 'config._id': { $in: deployingConfigs } }] + } + ); + } + + private async appendSyncConfigToStream(options: { + versioned: VersionedPowerSyncMongoV3; + existing: ReplicationStreamDocumentV3; + existingConfigDocs: SyncConfigDefinition[]; + options: storage.UpdateSyncRulesOptions; + storageVersion: number; + session: mongo.ClientSession; + }): Promise { + const { versioned, existing, existingConfigDocs, options: updateOptions, storageVersion, session } = options; + const existingConfigs = existingConfigDocs.map((doc) => ({ + plan: doc.serialized_plan!.plan, + mapping: BucketDefinitionMapping.fromSyncConfig(doc) + })); + const mapping = BucketDefinitionMapping.constructIncrementalMappingFromSerializedPlans( + existingConfigs, + updateOptions.config.plan!.plan + ); + + const syncConfigDoc: SyncConfigDefinition = { + _id: new ObjectId(), + replication_stream_id: existing._id, + created_at: new Date(), + storage_version: storageVersion, + content: updateOptions.config.yaml, + serialized_plan: updateOptions.config.plan, + rule_mapping: mapping.serialize() + }; + await versioned.syncConfigDefinitions.insertOne(syncConfigDoc, { session }); + const syncConfigState: SyncRuleConfigStateV3 = { + _id: syncConfigDoc._id, + state: storage.SyncRuleState.PROCESSING, + keepalive_op: null, + last_checkpoint: null, + last_checkpoint_lsn: null, + no_checkpoint_before: null, + snapshot_done: false + }; + + await this.db.sync_rules.updateOne( + { _id: existing._id }, + { + $push: { + sync_configs: syncConfigState + }, + $set: { + last_fatal_error: null, + last_fatal_error_ts: null + } + }, + { session } + ); + await this.db.notifyCheckpoint(); + const stream = new MongoPersistedReplicationStream( + this.db, + { + ...existing, + sync_configs: [...existing.sync_configs, syncConfigState] + }, + [...existingConfigDocs, syncConfigDoc] + ); + if (updateOptions.lock) { + await stream.lock(session); + } + return stream; + } + + async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise { const storageVersion = options.storageVersion ?? options.config.parsed.config.storageVersion ?? storage.CURRENT_STORAGE_VERSION; @@ -249,7 +441,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { return this.updateSyncRulesV3(options, storageVersion, storageConfig); } - let rules: MongoPersistedSyncRulesContentV1 | undefined = undefined; + let rules: MongoPersistedReplicationStream | undefined = undefined; const session = this.session; @@ -303,7 +495,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { await this.db.sync_rules.insertOne(doc, { session }); await this.db.notifyCheckpoint(); - rules = new MongoPersistedSyncRulesContentV1(this.db, doc); + rules = new MongoPersistedReplicationStream(this.db, doc); if (options.lock) { // The lock is persisted on rules.current_lock await rules.lock(session); @@ -313,8 +505,8 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { return rules!; } - async getActiveSyncRulesContent(): Promise< - MongoPersistedSyncRulesContentV1 | MongoPersistedSyncRulesContentV3 | null + async getActiveSyncConfigContent(): Promise< + MongoPersistedSyncConfigContentV1 | MongoPersistedSyncConfigContentV3 | null > { const doc = await this.db.sync_rules.findOne( { @@ -326,7 +518,30 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { return this.getSyncRulesContent(doc, [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED]); } + async getActiveSyncConfigStatus(): Promise { + const content = await this.getActiveSyncConfigContent(); + if (content == null) { + return null; + } + + return content.getSyncConfigStatus(); + } + private async getSyncRulesContent(doc: SyncRuleDocumentBase | null, stateFilter: storage.SyncRuleState[]) { + return (await this.getSyncRulesContents(doc, stateFilter))[0] ?? null; + } + + async getReplicationStream(replicationStreamId: number): Promise { + const doc = await this.db.sync_rules.findOne({ _id: replicationStreamId }); + return this.replicationStreamFromDoc(doc, [ + storage.SyncRuleState.PROCESSING, + storage.SyncRuleState.ACTIVE, + storage.SyncRuleState.ERRORED, + storage.SyncRuleState.STOP + ]); + } + + private async replicationStreamFromDoc(doc: SyncRuleDocumentBase | null, stateFilter: storage.SyncRuleState[]) { if (doc == null) { return null; } @@ -334,29 +549,62 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { if (storageConfig.incrementalReprocessing) { const v3 = doc as ReplicationStreamDocumentV3; - const active = v3.sync_configs.find((c) => stateFilter.includes(c.state)); - if (active == null) { + const matching = v3.sync_configs.filter((c) => stateFilter.includes(c.state)); + if (matching.length == 0) { + return null; + } + + const db = this.db.versioned(storageConfig) as VersionedPowerSyncMongoV3; + const syncConfigDocs = await db.syncConfigDefinitions + .find({ + _id: { $in: matching.map((config) => config._id) } + }) + .toArray(); + + if (syncConfigDocs.length == 0) { return null; } + return new MongoPersistedReplicationStream(this.db, v3, syncConfigDocs); + } + + return new MongoPersistedReplicationStream(this.db, doc as SyncRuleDocumentV1); + } + + private async getSyncRulesContents(doc: SyncRuleDocumentBase | null, stateFilter: storage.SyncRuleState[]) { + if (doc == null) { + return []; + } + const storageConfig = getMongoStorageConfig(doc.storage_version ?? LEGACY_STORAGE_VERSION); + + if (storageConfig.incrementalReprocessing) { + const v3 = doc as ReplicationStreamDocumentV3; + const matching = v3.sync_configs.filter((c) => stateFilter.includes(c.state)); + if (matching.length == 0) { + return []; + } // TODO: cache the config. It could specifically help for the main replication loop // that checks for active replication streams. // It is not a major bottleneck though, since it only runs once every couple of seconds at most. const db = this.db.versioned(storageConfig) as VersionedPowerSyncMongoV3; - const syncConfigDoc = await db.syncConfigDefinitions.findOne({ _id: active._id }); - if (syncConfigDoc == null) { - return null; - } - return new MongoPersistedSyncRulesContentV3(this.db, v3, syncConfigDoc); + const syncConfigDocs = await db.syncConfigDefinitions + .find({ + _id: { $in: matching.map((config) => config._id) } + }) + .toArray(); + + return syncConfigDocs.map((syncConfigDoc) => new MongoPersistedSyncConfigContentV3(this.db, v3, syncConfigDoc)); } - return new MongoPersistedSyncRulesContentV1(this.db, doc as SyncRuleDocumentV1); + return [new MongoPersistedSyncConfigContentV1(this.db, doc as SyncRuleDocumentV1)]; } - async getNextSyncRulesContent(): Promise { + async getDeployingSyncConfigContent(): Promise< + MongoPersistedSyncConfigContentV1 | MongoPersistedSyncConfigContentV3 | null + > { const doc = await this.db.sync_rules.findOne( { - state: storage.SyncRuleState.PROCESSING + $or: [{ state: storage.SyncRuleState.PROCESSING }, { 'sync_configs.state': storage.SyncRuleState.PROCESSING }] }, { sort: { _id: -1 }, limit: 1 } ); @@ -364,7 +612,59 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { return this.getSyncRulesContent(doc, [storage.SyncRuleState.PROCESSING]); } - async getReplicatingSyncRules(): Promise { + async getReplicationStreamConfigs( + replicationStreamId: number + ): Promise<(MongoPersistedSyncConfigContentV1 | MongoPersistedSyncConfigContentV3)[]> { + const doc = await this.db.sync_rules.findOne({ _id: replicationStreamId }); + if (doc == null) { + return []; + } + + return this.getSyncRulesContents(doc, [ + storage.SyncRuleState.PROCESSING, + storage.SyncRuleState.ACTIVE, + storage.SyncRuleState.ERRORED, + storage.SyncRuleState.STOP + ]); + } + + async getSyncConfigContent( + syncConfigId: storage.PersistedSyncConfigId + ): Promise { + if (ObjectId.isValid(syncConfigId)) { + const syncConfigObjectId = new ObjectId(syncConfigId); + const stream = await this.db.sync_rules.findOne({ + 'sync_configs._id': syncConfigObjectId + }); + if (stream != null) { + const storageConfig = getMongoStorageConfig(stream.storage_version ?? LEGACY_STORAGE_VERSION); + if (storageConfig.incrementalReprocessing) { + const db = this.db.versioned(storageConfig) as VersionedPowerSyncMongoV3; + const syncConfigDoc = await db.syncConfigDefinitions.findOne({ _id: syncConfigObjectId }); + if (syncConfigDoc != null) { + return new MongoPersistedSyncConfigContentV3(this.db, stream, syncConfigDoc); + } + } + } + } + + const replicationStreamId = Number(syncConfigId); + if (!Number.isInteger(replicationStreamId)) { + return null; + } + + const doc = await this.db.sync_rules.findOne({ _id: replicationStreamId }); + if (doc == null) { + return null; + } + const storageConfig = getMongoStorageConfig(doc.storage_version ?? LEGACY_STORAGE_VERSION); + if (storageConfig.incrementalReprocessing) { + return null; + } + return new MongoPersistedSyncConfigContentV1(this.db, doc as SyncRuleDocumentV1); + } + + async getReplicatingReplicationStreams(): Promise { const docs = await this.db.sync_rules .find({ state: { $in: [storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE] } @@ -374,13 +674,13 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { return ( await Promise.all( docs.map((doc) => { - return this.getSyncRulesContent(doc, [storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE]); + return this.replicationStreamFromDoc(doc, [storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE]); }) ) ).filter((r) => r != null); } - async getStoppedSyncRules(): Promise { + async getStoppedReplicationStreams(): Promise { const docs = await this.db.sync_rules .find({ state: storage.SyncRuleState.STOP @@ -390,25 +690,34 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { return ( await Promise.all( docs.map((doc) => { - return this.getSyncRulesContent(doc, [storage.SyncRuleState.STOP]); + return this.replicationStreamFromDoc(doc, [storage.SyncRuleState.STOP]); }) ) ).filter((d) => d != null); } async getActiveStorage(): Promise { - const content = await this.getActiveSyncRulesContent(); - if (content == null) { + const doc = await this.db.sync_rules.findOne( + { + state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] } + }, + { sort: { _id: -1 }, limit: 1 } + ); + const stream = await this.replicationStreamFromDoc(doc, [ + storage.SyncRuleState.ACTIVE, + storage.SyncRuleState.ERRORED + ]); + if (stream == null) { return null; } // It is important that this instance is cached. // Not for the instance construction itself, but to ensure that internal caches on the instance // are re-used properly. - if (this.activeStorageCache?.group_id == content.id) { + if (this.activeStorageCache?.group_id == stream.id) { return this.activeStorageCache; } else { - const instance = this.getInstance(content); + const instance = this.getInstance(stream); this.activeStorageCache = instance; return instance; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts b/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts index df33e4e51..bb49de135 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/BucketDefinitionMapping.ts @@ -2,20 +2,49 @@ import { ServiceAssertionError } from '@powersync/lib-services-framework'; import { BucketDataSource, BucketDefinitionId, + HashMap, ParameterIndexId, ParameterIndexLookupCreator, + ParameterLookupDefinitionId, + SerializedBucketDataSourceWithDataSources, + SerializedParameterIndexLookupCreator, + serializedStreamBucketDataSourceEquality, + serializedStreamParameterIndexLookupCreatorEquality, + SerializedSyncPlanV1, SyncConfigWithErrors } from '@powersync/service-sync-rules'; import { SyncConfigDefinition } from '../storage-index.js'; +export interface SerializedSyncConfigWithMapping { + plan: SerializedSyncPlanV1; + mapping: BucketDefinitionMapping; +} + +export interface SyncConfigWithMapping { + syncConfig: SyncConfigWithErrors; + mapping: BucketDefinitionMapping | null; +} + +export interface SyncConfigWithRequiredMapping { + syncConfigId?: string; + syncConfig: SyncConfigWithErrors; + mapping: BucketDefinitionMapping; +} + +/** + * Represents a mapping from bucket data sources and parameter lookup sources to stable IDs used for bucket definition and parameter index persistence. + * + * An instance of BucketDefinitionMapping is associated with a specific SyncConfig. + * MongoHydrationState handles the mapping across multiple SyncConfigs in the same replication stream. + */ export class BucketDefinitionMapping { static fromSyncConfig(doc: Pick): BucketDefinitionMapping { return new BucketDefinitionMapping(doc.rule_mapping?.definitions ?? {}, doc.rule_mapping?.parameter_indexes ?? {}); } - static fromParsedSyncRules(syncRules: SyncConfigWithErrors): BucketDefinitionMapping { - const definitionNames = syncRules.config.bucketDataSources.map((source) => source.uniqueName).sort(); - const parameterKeys = syncRules.config.bucketParameterLookupSources + static fromParsedSyncConfig(syncConfig: SyncConfigWithErrors): BucketDefinitionMapping { + const definitionNames = syncConfig.config.bucketDataSources.map((source) => source.uniqueName).sort(); + const parameterKeys = syncConfig.config.bucketParameterLookupSources .map((source) => `${source.sourceId.lookupName}#${source.sourceId.queryId}`) .sort(); @@ -32,15 +61,88 @@ export class BucketDefinitionMapping { return new BucketDefinitionMapping(definitions, parameterLookups); } + static constructIncrementalMappingFromSerializedPlans( + existing: SerializedSyncConfigWithMapping[], + newPlan: SerializedSyncPlanV1 + ): BucketDefinitionMapping { + // FIXME: These ids may conflict with existing mappings if sync configs are de-activated. + let nextBucketDefinitionId = + existing + .map((c) => c.mapping.allBucketDefinitionIds()) + .flat() + .reduce((maxId, id) => Math.max(maxId, parseInt(id, 16)), 0) + 1; + function generateNewBucketDefinitionId(): BucketDefinitionId { + const id = nextBucketDefinitionId.toString(16); + nextBucketDefinitionId++; + return id; + } + let nextParameterIndexId = + existing + .map((c) => c.mapping.allParameterIndexIds()) + .flat() + .reduce((maxId, id) => Math.max(maxId, parseInt(id, 16)), 0) + 1; + function generateNewParameterIndexId(): ParameterIndexId { + const id = nextParameterIndexId.toString(16); + nextParameterIndexId++; + return id; + } + + const definitions: Record = {}; + const parameterLookups: Record = {}; + const compatibleBuckets = new HashMap( + serializedStreamBucketDataSourceEquality + ); + const compatibleParameterLookups = new HashMap( + serializedStreamParameterIndexLookupCreatorEquality + ); + + for (const config of existing) { + for (const bucket of config.plan.buckets) { + compatibleBuckets.putIfAbsent({ bucket, dataSources: config.plan.dataSources }, () => + config.mapping.bucketSourceIdByName(bucket.uniqueName) + ); + } + + for (const parameterLookup of config.plan.parameterIndexes) { + compatibleParameterLookups.putIfAbsent(parameterLookup, () => + config.mapping.parameterLookupIdByKey(parameterLookupKey(parameterLookup.lookupScope)) + ); + } + } + + for (const bucket of newPlan.buckets) { + const compatibleId = compatibleBuckets.get({ bucket, dataSources: newPlan.dataSources }); + const id = compatibleId ?? generateNewBucketDefinitionId(); + definitions[bucket.uniqueName] = id; + } + + for (const parameterLookup of newPlan.parameterIndexes) { + const compatibleId = compatibleParameterLookups.get(parameterLookup); + const id = compatibleId ?? generateNewParameterIndexId(); + parameterLookups[parameterLookupKey(parameterLookup.lookupScope)] = id; + } + + return new BucketDefinitionMapping(definitions, parameterLookups); + } + constructor( private definitions: Record = {}, private parameterLookupMapping: Record = {} ) {} + /** + * Given a BucketDataSource within this SyncConfig, return the BucketDefinitionId, or throw if not found. + * + * The behavior is undefined if the source is associated with a different SyncConfig. + */ bucketSourceId(source: BucketDataSource): BucketDefinitionId { - const defId = this.definitions[source.uniqueName]; + return this.bucketSourceIdByName(source.uniqueName); + } + + bucketSourceIdByName(uniqueName: string): BucketDefinitionId { + const defId = this.definitions[uniqueName]; if (defId == null) { - throw new ServiceAssertionError(`No mapping found for bucket source ${source.uniqueName}`); + throw new ServiceAssertionError(`No mapping found for bucket source ${uniqueName}`); } return defId; } @@ -54,7 +156,10 @@ export class BucketDefinitionMapping { } parameterLookupId(source: ParameterIndexLookupCreator): ParameterIndexId { - const key = this.parameterLookupKey(source.sourceId.lookupName, source.sourceId.queryId); + return this.parameterLookupIdByKey(parameterLookupKey(source.sourceId)); + } + + parameterLookupIdByKey(key: string): ParameterIndexId { const defId = this.parameterLookupMapping[key]; if (defId == null) { throw new ServiceAssertionError(`No mapping found for parameter lookup source ${key}`); @@ -62,10 +167,6 @@ export class BucketDefinitionMapping { return defId; } - private parameterLookupKey(lookupName: string, queryId: string) { - return `${lookupName}#${queryId}`; - } - serialize(): SyncConfigDefinition['rule_mapping'] { return { definitions: { ...this.definitions }, @@ -73,3 +174,90 @@ export class BucketDefinitionMapping { }; } } + +export class MultiSyncConfigBucketDefinitionMapping extends BucketDefinitionMapping { + private bucketDataSourceMappings = new WeakMap(); + private bucketDataSourceMappingsByName = new Map(); + private parameterLookupMappings = new WeakMap(); + private parameterLookupMappingsByKey = new Map(); + private mappings: BucketDefinitionMapping[]; + + constructor(syncConfigs: SyncConfigWithRequiredMapping[]) { + super(); + this.mappings = syncConfigs.map((config) => config.mapping); + + for (const config of syncConfigs) { + for (const source of config.syncConfig.config.bucketDataSources) { + this.bucketDataSourceMappings.set(source, config.mapping); + addMappingEntry(this.bucketDataSourceMappingsByName, source.uniqueName, config); + } + for (const source of config.syncConfig.config.bucketParameterLookupSources) { + this.parameterLookupMappings.set(source, config.mapping); + addMappingEntry(this.parameterLookupMappingsByKey, parameterLookupKey(source.sourceId), config); + } + } + } + + bucketSourceId(source: BucketDataSource): BucketDefinitionId { + const mapping = this.bucketDataSourceMappings.get(source); + if (mapping != null) { + return mapping.bucketSourceId(source); + } + + const id = this.unambiguousBucketSourceIdByName(source.uniqueName); + if (id == null) { + throw new ServiceAssertionError(`No mapping found for bucket source ${source.uniqueName}`); + } + return id; + } + + allBucketDefinitionIds(): BucketDefinitionId[] { + return [...new Set(this.mappings.flatMap((mapping) => mapping.allBucketDefinitionIds()))]; + } + + parameterLookupId(source: ParameterIndexLookupCreator): ParameterIndexId { + const mapping = this.parameterLookupMappings.get(source); + if (mapping != null) { + return mapping.parameterLookupId(source); + } + + const key = parameterLookupKey(source.sourceId); + const id = this.unambiguousParameterLookupIdByKey(key); + if (id == null) { + throw new ServiceAssertionError( + `No mapping found for parameter lookup source ${source.sourceId.lookupName}#${source.sourceId.queryId}` + ); + } + return id; + } + + allParameterIndexIds(): ParameterIndexId[] { + return [...new Set(this.mappings.flatMap((mapping) => mapping.allParameterIndexIds()))]; + } + + private unambiguousBucketSourceIdByName(uniqueName: string): BucketDefinitionId | null { + const entries = this.bucketDataSourceMappingsByName.get(uniqueName) ?? []; + const ids = new Set(entries.map((entry) => entry.mapping.bucketSourceIdByName(uniqueName))); + return ids.size == 1 ? [...ids][0] : null; + } + + private unambiguousParameterLookupIdByKey(key: string): ParameterIndexId | null { + const entries = this.parameterLookupMappingsByKey.get(key) ?? []; + const ids = new Set(entries.map((entry) => entry.mapping.parameterLookupIdByKey(key))); + return ids.size == 1 ? [...ids][0] : null; + } +} + +export function parameterLookupKey(id: ParameterLookupDefinitionId) { + return `${id.lookupName}#${id.queryId}`; +} + +function addMappingEntry( + map: Map, + key: string, + config: SyncConfigWithRequiredMapping +) { + const existing = map.get(key) ?? []; + existing.push(config); + map.set(key, existing); +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index 9c3e18d60..2b39290c3 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -44,7 +44,7 @@ export interface MongoBucketBatchOptions { syncRules: HydratedSyncConfig; groupId: number; slotName: string; - syncConfigId?: bson.ObjectId | null; + syncConfigIds?: bson.ObjectId[]; lastCheckpointLsn: string | null; keepaliveOp: InternalOpId | null; resumeFromLsn: string | null; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts index c0cc765f7..29af40fdf 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRules.ts @@ -12,67 +12,127 @@ import { nodeSqlite, ParameterIndexLookupCreator, ParameterLookupScope, - SyncConfigWithErrors, versionedHydrationState } from '@powersync/service-sync-rules'; -import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; +import { + BucketDefinitionMapping, + MultiSyncConfigBucketDefinitionMapping, + SyncConfigWithMapping, + SyncConfigWithRequiredMapping +} from './BucketDefinitionMapping.js'; import { StorageConfig } from './models.js'; export class MongoPersistedSyncRules implements storage.PersistedSyncRules { public readonly hydrationState: HydrationState; + public readonly syncConfigs: storage.PersistedSyncRules['syncConfigs']; + public readonly slot_name: string; + public readonly mapping: BucketDefinitionMapping; constructor( public readonly id: number, - public readonly syncConfigWithErrors: SyncConfigWithErrors, - public readonly slot_name: string, - private readonly mapping: BucketDefinitionMapping | null, - private readonly storageConfig: StorageConfig + storageConfig: StorageConfig, + slotName: string, + syncConfigs: SyncConfigWithMapping[] ) { - if (this.storageConfig.incrementalReprocessing) { - if (this.mapping == null) { + this.slot_name = slotName; + this.syncConfigs = syncConfigs.map((config) => config.syncConfig); + if (this.syncConfigs.length == 0) { + throw new ServiceAssertionError(`At least one sync config is required`); + } + const [firstConfig] = this.syncConfigs; + const compatibility = firstConfig.config.compatibility; + for (const config of this.syncConfigs) { + if (config.config.compatibility.equals(compatibility)) { + continue; + } + throw new ServiceAssertionError( + `All sync configs in a replication stream must use the same compatibility options` + ); + } + + if (storageConfig.incrementalReprocessing) { + if (syncConfigs.some((c) => c.mapping == null)) { throw new ServiceAssertionError(`mapping is required for v3 storage`); } - this.hydrationState = new MongoHydrationState(this.mapping, this.id); - } else if ( - !this.syncConfigWithErrors.config.compatibility.isEnabled(CompatibilityOption.versionedBucketIds) && - !this.storageConfig.versionedBuckets - ) { + const mappedConfigs = syncConfigs as SyncConfigWithRequiredMapping[]; + this.hydrationState = new MongoHydrationState(mappedConfigs, this.id); + this.mapping = new MultiSyncConfigBucketDefinitionMapping(mappedConfigs); + } else if (!compatibility.isEnabled(CompatibilityOption.versionedBucketIds) && !storageConfig.versionedBuckets) { + const [syncConfig] = syncConfigs; + if (syncConfigs.length != 1 || syncConfig == null) { + throw new ServiceAssertionError(`Non-incremental storage requires exactly one sync config`); + } this.hydrationState = DEFAULT_HYDRATION_STATE; + this.mapping = syncConfig.mapping ?? new BucketDefinitionMapping(); } else { + const [syncConfig] = syncConfigs; + if (syncConfigs.length != 1 || syncConfig == null) { + throw new ServiceAssertionError(`Non-incremental storage requires exactly one sync config`); + } this.hydrationState = versionedHydrationState(this.id); + this.mapping = syncConfig.mapping ?? new BucketDefinitionMapping(); } } hydratedSyncConfig(): HydratedSyncConfig { - return this.syncConfigWithErrors.config.hydrate({ - hydrationState: this.hydrationState, - sqlite: nodeSqlite(sqlite) + return new HydratedSyncConfig({ + definitions: this.syncConfigs.map((config) => config.config), + createParams: { + hydrationState: this.hydrationState, + sqlite: nodeSqlite(sqlite) + } }); } } class MongoHydrationState implements HydrationState { + private bucketDataSourceSyncConfig = new WeakMap(); + private parameterIndexLookupSyncConfig = new WeakMap(); + constructor( - private readonly mapping: BucketDefinitionMapping, + readonly syncConfigs: SyncConfigWithRequiredMapping[], private readonly version: number - ) {} + ) { + for (let syncConfig of syncConfigs) { + for (let source of syncConfig.syncConfig.config.bucketDataSources) { + this.bucketDataSourceSyncConfig.set(source, syncConfig); + } + for (let source of syncConfig.syncConfig.config.bucketParameterLookupSources) { + this.parameterIndexLookupSyncConfig.set(source, syncConfig); + } + } + } getBucketSourceScope(source: BucketDataSource): BucketDataScope { - // Keep this aligned with versionedHydrationState() for now. - // + const syncConfig = this.bucketDataSourceSyncConfig.get(source); + if (syncConfig == null) { + throw new ServiceAssertionError(`No sync config found for bucket data source ${source.uniqueName}`); + } + const mapping = syncConfig.mapping; + const defId = mapping.bucketSourceId(source); // Previous Mongo-specific behavior: // return { // bucketPrefix: defId, // source // }; + // FIXME: Should this use defId, or is uniqueName constant? return { + // Keep this aligned with versionedHydrationState() for now. + // May consider changing the format before stable release, e.g. bucketPrefix: defId bucketPrefix: `${this.version}#${source.uniqueName}`, source }; } getParameterIndexLookupScope(source: ParameterIndexLookupCreator): ParameterLookupScope { - const defId = this.mapping.parameterLookupId(source); + const syncConfig = this.parameterIndexLookupSyncConfig.get(source); + if (syncConfig == null) { + throw new ServiceAssertionError( + `No sync config found for parameter index lookup source ${source.sourceId.lookupName}#${source.sourceId.queryId}` + ); + } + const mapping = syncConfig.mapping; + const defId = mapping.parameterLookupId(source); return { lookupName: defId, queryId: '', diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts index f602493b4..439e6e12b 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoPersistedSyncRulesContent.ts @@ -10,22 +10,72 @@ import { PowerSyncMongo } from './db.js'; import { getMongoStorageConfig } from './models.js'; import { SyncRuleDocumentV1 } from './v1/models.js'; -abstract class MongoPersistedSyncRulesContentBase extends storage.PersistedSyncRulesContent { +export class MongoPersistedReplicationStream extends storage.PersistedReplicationStream { public current_lock: MongoSyncRulesLock | null = null; + + constructor( + private readonly db: PowerSyncMongo, + private readonly doc: SyncRuleDocumentV1 | ReplicationStreamDocumentV3, + private readonly configs: SyncConfigDefinition[] = [] + ) { + const storageVersion = doc.storage_version ?? storage.LEGACY_STORAGE_VERSION; + const replicationJobId = + configs.length == 0 + ? String(doc._id) + : `${doc._id}:${configs + .map((config) => config._id.toHexString()) + .sort() + .join(',')}`; + + super({ + id: doc._id, + slot_name: doc.slot_name ?? `powersync_${doc._id}`, + state: doc.state, + storageVersion, + replicationJobId + }); + } + + getStorageConfig() { + return getMongoStorageConfig(this.storageVersion); + } + + toSyncConfigContent(): MongoPersistedSyncConfigContentV1 | MongoPersistedSyncConfigContentV3 { + if (this.getStorageConfig().incrementalReprocessing) { + if (this.configs.length == 0) { + throw new ServiceAssertionError(`Cannot create v3 storage without sync config definitions`); + } + return new MongoPersistedSyncConfigContentV3(this.db, this.doc as ReplicationStreamDocumentV3, this.configs); + } + + return new MongoPersistedSyncConfigContentV1(this.db, this.doc as SyncRuleDocumentV1); + } + + async lock(session?: mongo.ClientSession) { + const lock = await MongoSyncRulesLock.createLock(this.db.versioned(this.getStorageConfig()), this, session); + this.current_lock = lock; + return lock; + } +} + +abstract class MongoPersistedSyncConfigContentBase extends storage.PersistedSyncConfigContent { public readonly mapping: BucketDefinitionMapping; - public readonly syncConfigId: bson.ObjectId | null; + public readonly syncConfigObjectId: bson.ObjectId | null; protected constructor( protected readonly db: PowerSyncMongo, - options: ConstructorParameters[0] & { + options: Omit & { mapping: BucketDefinitionMapping; syncConfigId: bson.ObjectId | null; } ) { const { mapping, syncConfigId, ...base } = options; - super(base); + super({ + ...base, + syncConfigId: syncConfigId?.toHexString() ?? null + }); this.mapping = mapping; - this.syncConfigId = syncConfigId; + this.syncConfigObjectId = syncConfigId; } getStorageConfig() { @@ -35,24 +85,18 @@ abstract class MongoPersistedSyncRulesContentBase extends storage.PersistedSyncR parsed(options: storage.ParseSyncRulesOptions): storage.PersistedSyncRules { const parsed = super.parsed(options); const storageConfig = this.getStorageConfig(); + const [syncConfig] = parsed.syncConfigs; + if (syncConfig == null) { + throw new ServiceAssertionError(`Expected one parsed sync config`); + } - return new MongoPersistedSyncRules( - parsed.id, - parsed.syncConfigWithErrors, - parsed.slot_name, - storageConfig.incrementalReprocessing ? this.mapping : null, - storageConfig - ); - } - - async lock(session?: mongo.ClientSession) { - const lock = await MongoSyncRulesLock.createLock(this.db.versioned(this.getStorageConfig()), this, session); - this.current_lock = lock; - return lock; + return new MongoPersistedSyncRules(parsed.id, storageConfig, parsed.slot_name, [ + { syncConfig, mapping: storageConfig.incrementalReprocessing ? this.mapping : null } + ]); } } -export class MongoPersistedSyncRulesContentV1 extends MongoPersistedSyncRulesContentBase { +export class MongoPersistedSyncConfigContentV1 extends MongoPersistedSyncConfigContentBase { constructor(db: PowerSyncMongo, doc: SyncRuleDocumentV1) { super(db, { id: doc._id, @@ -66,6 +110,7 @@ export class MongoPersistedSyncRulesContentV1 extends MongoPersistedSyncRulesCon last_checkpoint_ts: doc.last_checkpoint_ts, last_keepalive_ts: doc.last_keepalive_ts, active: doc.state == SyncRuleState.ACTIVE, + state: doc.state, storageVersion: doc.storage_version ?? storage.LEGACY_STORAGE_VERSION, mapping: new BucketDefinitionMapping(), syncConfigId: null @@ -73,18 +118,31 @@ export class MongoPersistedSyncRulesContentV1 extends MongoPersistedSyncRulesCon } } -export class MongoPersistedSyncRulesContentV3 extends MongoPersistedSyncRulesContentBase { - declare public readonly syncConfigId: bson.ObjectId; +export class MongoPersistedSyncConfigContentV3 extends MongoPersistedSyncConfigContentBase { + declare public readonly syncConfigObjectId: bson.ObjectId; + private readonly doc: ReplicationStreamDocumentV3; + private readonly configs: SyncConfigDefinition[]; + public readonly syncConfigIds: bson.ObjectId[]; - constructor(db: PowerSyncMongo, doc: ReplicationStreamDocumentV3, config: SyncConfigDefinition) { - const state = doc.sync_configs.find((c) => c._id.equals(config._id)); + constructor( + db: PowerSyncMongo, + doc: ReplicationStreamDocumentV3, + config: SyncConfigDefinition | SyncConfigDefinition[] + ) { + const configs = Array.isArray(config) ? config : [config]; + const selected = configs[0]; + const replicationJobId = `${doc._id}:${configs + .map((config) => config._id.toHexString()) + .sort() + .join(',')}`; + const state = doc.sync_configs.find((c) => c._id.equals(selected._id)); if (state == null) { - throw new ServiceAssertionError(`Cannot find sync config ${config._id} in replication stream ${doc._id}`); + throw new ServiceAssertionError(`Cannot find sync config ${selected._id} in replication stream ${doc._id}`); } super(db, { id: doc._id, - sync_rules_content: config.content, - compiled_plan: config.serialized_plan ?? null, + sync_rules_content: selected.content, + compiled_plan: selected.serialized_plan ?? null, last_checkpoint_lsn: state?.last_checkpoint_lsn ?? null, slot_name: doc.slot_name ?? `powersync_${doc._id}`, @@ -93,9 +151,33 @@ export class MongoPersistedSyncRulesContentV3 extends MongoPersistedSyncRulesCon last_checkpoint_ts: doc.last_checkpoint_ts, last_keepalive_ts: doc.last_keepalive_ts, active: doc.state == SyncRuleState.ACTIVE && state.state == SyncRuleState.ACTIVE, + state: state.state, storageVersion: doc.storage_version, - mapping: BucketDefinitionMapping.fromSyncConfig(config), - syncConfigId: config._id + mapping: BucketDefinitionMapping.fromSyncConfig(selected), + syncConfigId: selected._id, + replicationJobId }); + this.doc = doc; + this.configs = configs; + this.syncConfigIds = configs.map((config) => config._id); + } + + parsed(options: storage.ParseSyncRulesOptions): storage.PersistedSyncRules { + const storageConfig = this.getStorageConfig(); + const syncConfigs = this.configs.map((config) => { + const content = new MongoPersistedSyncConfigContentV3(this.db, this.doc, config); + const parsed = storage.PersistedSyncConfigContent.prototype.parsed.call(content, options); + const [syncConfig] = parsed.syncConfigs; + if (syncConfig == null) { + throw new ServiceAssertionError(`Expected one parsed sync config`); + } + return { + syncConfigId: config._id.toHexString(), + syncConfig, + mapping: BucketDefinitionMapping.fromSyncConfig(config) + }; + }); + + return new MongoPersistedSyncRules(this.id, storageConfig, this.slot_name, syncConfigs); } } diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 1928e41c5..a27ad207e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -27,6 +27,7 @@ import { LRUCache } from 'lru-cache'; import * as timers from 'timers/promises'; import { retryOnMongoMaxTimeMSExpired } from '../../utils/util.js'; import { MongoBucketStorage } from '../MongoBucketStorage.js'; +import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; import { MongoSyncBucketStorageContext } from './common/MongoSyncBucketStorageContext.js'; import type { VersionedPowerSyncMongo } from './db.js'; import { StorageConfig } from './models.js'; @@ -34,7 +35,7 @@ import { MongoBucketBatchOptions } from './MongoBucketBatch.js'; import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js'; import { MongoCompactOptions, MongoCompactor } from './MongoCompactor.js'; import { MongoParameterCompactor } from './MongoParameterCompactor.js'; -import { MongoPersistedSyncRulesContentV1 } from './MongoPersistedSyncRulesContent.js'; +import { MongoPersistedSyncConfigContentV1 } from './MongoPersistedSyncRulesContent.js'; import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js'; export interface MongoSyncBucketStorageOptions { @@ -51,7 +52,7 @@ interface WriterSyncState { lastCheckpointLsn: string | null; resumeFromLsn: string | null; keepaliveOp: InternalOpId | null; - syncConfigId?: bson.ObjectId | null; + syncConfigIds?: bson.ObjectId[]; } /** @@ -83,7 +84,7 @@ export abstract class MongoSyncBucketStorage constructor( public readonly factory: MongoBucketStorage, public readonly group_id: number, - protected readonly sync_rules: MongoPersistedSyncRulesContentV1, + protected readonly sync_rules: MongoPersistedSyncConfigContentV1, public readonly slot_name: string, writeCheckpointMode: storage.WriteCheckpointMode | undefined, options: MongoSyncBucketStorageOptions @@ -195,12 +196,15 @@ export abstract class MongoSyncBucketStorage await this.initializeStorage(); const state = await this.getWriterSyncState(); + const parsed = this.sync_rules.parsed(options) as storage.PersistedSyncRules & { + mapping?: BucketDefinitionMapping; + }; const batchOptions: MongoBucketBatchOptions = { logger: options.logger ?? this.logger, db: this.db, - syncRules: this.sync_rules.parsed(options).hydratedSyncConfig(), - mapping: this.sync_rules.mapping, + syncRules: parsed.hydratedSyncConfig(), + mapping: parsed.mapping ?? this.sync_rules.mapping, groupId: this.group_id, slotName: this.slot_name, lastCheckpointLsn: state.lastCheckpointLsn, @@ -210,7 +214,7 @@ export abstract class MongoSyncBucketStorage skipExistingRows: options.skipExistingRows ?? false, markRecordUnavailable: options.markRecordUnavailable, hooks: options.hooks, - syncConfigId: state.syncConfigId, + syncConfigIds: state.syncConfigIds, tracer: options.tracer }; const writer = this.createWriterImpl(batchOptions); diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncRulesLock.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncRulesLock.ts index 31410fcce..fc3f8ad16 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncRulesLock.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncRulesLock.ts @@ -17,7 +17,7 @@ export class MongoSyncRulesLock implements storage.ReplicationLock { */ static async createLock( db: VersionedPowerSyncMongo, - sync_rules: storage.PersistedSyncRulesContent, + sync_rules: storage.PersistedReplicationStream, session?: mongo.ClientSession ): Promise { const lockId = crypto.randomBytes(8).toString('hex'); diff --git a/modules/module-mongodb-storage/src/storage/implementation/createMongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/createMongoSyncBucketStorage.ts index bd5e7b810..8dcdb05e1 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/createMongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/createMongoSyncBucketStorage.ts @@ -1,6 +1,6 @@ import { storage } from '@powersync/service-core'; import { MongoBucketStorage } from '../MongoBucketStorage.js'; -import { MongoPersistedSyncRulesContentV1 } from './MongoPersistedSyncRulesContent.js'; +import { MongoPersistedSyncConfigContentV1 } from './MongoPersistedSyncRulesContent.js'; import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from './MongoSyncBucketStorage.js'; import { MongoSyncBucketStorageV1 } from './v1/MongoSyncBucketStorageV1.js'; import { MongoSyncBucketStorageV3 } from './v3/MongoSyncBucketStorageV3.js'; @@ -12,7 +12,7 @@ export type { MongoSyncBucketStorage }; export function createMongoSyncBucketStorage( factory: MongoBucketStorage, group_id: number, - sync_rules: MongoPersistedSyncRulesContentV1, + sync_rules: MongoPersistedSyncConfigContentV1, slot_name: string, writeCheckpointMode: storage.WriteCheckpointMode | undefined, options: MongoSyncBucketStorageOptions diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index f8c6a2d46..d19958086 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -27,7 +27,7 @@ import { MongoBucketBatchOptions } from '../MongoBucketBatch.js'; import { MongoChecksums } from '../MongoChecksums.js'; import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; -import { MongoPersistedSyncRulesContentV1 } from '../MongoPersistedSyncRulesContent.js'; +import { MongoPersistedSyncConfigContentV1 } from '../MongoPersistedSyncRulesContent.js'; import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js'; import { BucketDataDocumentV1, @@ -50,7 +50,7 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage { constructor( factory: MongoBucketStorage, group_id: number, - sync_rules: MongoPersistedSyncRulesContentV1, + sync_rules: MongoPersistedSyncConfigContentV1, slot_name: string, writeCheckpointMode: storage.WriteCheckpointMode | undefined, options: MongoSyncBucketStorageOptions @@ -94,8 +94,7 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage { return { lastCheckpointLsn: checkpointLsn, resumeFromLsn: maxLsn(checkpointLsn, doc?.snapshot_lsn), - keepaliveOp: doc?.keepalive_op ? BigInt(doc.keepalive_op) : null, - syncConfigId: null + keepaliveOp: doc?.keepalive_op ? BigInt(doc.keepalive_op) : null }; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts index cffdb1e78..dfb6642fc 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts @@ -22,16 +22,17 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { declare public readonly db: VersionedPowerSyncMongoV3; private readonly store: SourceRecordStore; - private readonly syncConfigId: bson.ObjectId; + private readonly syncConfigIds: bson.ObjectId[]; private needsActivationV3 = true; private lastWaitingLogThrottledV3 = 0; constructor(options: MongoBucketBatchOptions) { super(options); - if (options.syncConfigId == null) { + const syncConfigIds = options.syncConfigIds ?? []; + if (syncConfigIds.length == 0) { throw new ReplicationAssertionError('Missing sync config id for v3 batch'); } - this.syncConfigId = options.syncConfigId; + this.syncConfigIds = syncConfigIds; this.store = new SourceRecordStoreV3(this.db, this.group_id, this.mapping); } @@ -313,94 +314,108 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { const preUpdateDocument = await this.db.sync_rules.findOne( { _id: this.group_id, - 'sync_configs._id': this.syncConfigId + 'sync_configs._id': { $in: this.syncConfigIds } }, { session: this.session, projection: { snapshot_lsn: 1, - sync_configs: { - $elemMatch: { - _id: this.syncConfigId - } - } + sync_configs: 1 } } ); - const state = (preUpdateDocument as ReplicationStreamDocumentV3)?.sync_configs?.[0]; - if (state == null) { + const states = + (preUpdateDocument as ReplicationStreamDocumentV3)?.sync_configs?.filter((config) => + this.syncConfigIds.some((id) => id.equals(config._id)) + ) ?? []; + if (states.length == 0) { throw new ReplicationAssertionError( - `Failed to update checkpoint - no matching sync_config for _id: ${this.group_id}/${this.syncConfigId.toHexString()}` + `Failed to update checkpoint - no matching sync_config for _id: ${this.group_id}/${this.syncConfigIds + .map((id) => id.toHexString()) + .join(',')}` ); } - const checkpointState = calculateCheckpointState({ - lsn, - snapshotDone: state.snapshot_done === true, - lastCheckpointLsn: state.last_checkpoint_lsn, - noCheckpointBefore: state.no_checkpoint_before, - keepaliveOp: state.keepalive_op == null ? null : BigInt(state.keepalive_op), - lastCheckpoint: state.last_checkpoint, - persistedOp: this.persisted_op, - createEmptyCheckpoints - }); - - const updateSet: Record = { - last_keepalive_ts: now, - last_fatal_error: null, - last_fatal_error_ts: null, - 'sync_configs.$[config].keepalive_op': checkpointState.newKeepaliveOp, - 'sync_configs.$[config].last_checkpoint': checkpointState.newLastCheckpoint - }; - if (checkpointState.checkpointCreated) { - updateSet['sync_configs.$[config].last_checkpoint_lsn'] = lsn; - updateSet['snapshot_lsn'] = null; - updateSet['last_checkpoint_ts'] = now; - } + let checkpointBlocked = false; + let checkpointCreated = false; + let checkpointLogState: unknown = null; + const cleanupCheckpoints: bigint[] = []; + + for (const state of states) { + const checkpointState = calculateCheckpointState({ + lsn, + snapshotDone: state.snapshot_done === true, + lastCheckpointLsn: state.last_checkpoint_lsn, + noCheckpointBefore: state.no_checkpoint_before, + keepaliveOp: state.keepalive_op == null ? null : BigInt(state.keepalive_op), + lastCheckpoint: state.last_checkpoint, + persistedOp: this.persisted_op, + createEmptyCheckpoints + }); + + checkpointBlocked ||= checkpointState.checkpointBlocked; + checkpointCreated ||= checkpointState.checkpointCreated; + if (checkpointState.newLastCheckpoint != null) { + cleanupCheckpoints.push(checkpointState.newLastCheckpoint); + } + checkpointLogState ??= { + snapshot_done: state.snapshot_done, + last_checkpoint_lsn: state.last_checkpoint_lsn, + no_checkpoint_before: state.no_checkpoint_before + }; - await this.db.sync_rules.updateOne( - { - _id: this.group_id, - 'sync_configs._id': this.syncConfigId - }, - { - $set: updateSet - }, - { - session: this.session, - arrayFilters: [{ 'config._id': this.syncConfigId }] + const updateSet: Record = { + last_keepalive_ts: now, + last_fatal_error: null, + last_fatal_error_ts: null, + 'sync_configs.$[config].keepalive_op': checkpointState.newKeepaliveOp, + 'sync_configs.$[config].last_checkpoint': checkpointState.newLastCheckpoint + }; + if (checkpointState.checkpointCreated) { + updateSet['sync_configs.$[config].last_checkpoint_lsn'] = lsn; + updateSet['snapshot_lsn'] = null; + updateSet['last_checkpoint_ts'] = now; } - ); - if (checkpointState.checkpointBlocked) { + await this.db.sync_rules.updateOne( + { + _id: this.group_id, + 'sync_configs._id': state._id + }, + { + $set: updateSet + }, + { + session: this.session, + arrayFilters: [{ 'config._id': state._id }] + } + ); + } + + if (checkpointBlocked) { if (Date.now() - this.lastWaitingLogThrottledV3 > 5_000) { this.logger.info( - `Waiting before creating checkpoint, currently at ${lsn} / ${checkpointState.newKeepaliveOp}. Current state: ${JSON.stringify( - { - snapshot_done: state.snapshot_done, - last_checkpoint_lsn: state.last_checkpoint_lsn, - no_checkpoint_before: state.no_checkpoint_before - } - )}` + `Waiting before creating checkpoint, currently at ${lsn}. Current state: ${JSON.stringify(checkpointLogState)}` ); this.lastWaitingLogThrottledV3 = Date.now(); } } else { - if (checkpointState.checkpointCreated) { - this.logger.debug(`Created checkpoint at ${lsn} / ${checkpointState.newLastCheckpoint}`); + if (checkpointCreated) { + this.logger.debug(`Created checkpoint at ${lsn}`); } await this.autoActivateV3(lsn); await this.db.notifyCheckpoint(); this.persisted_op = null; this.last_checkpoint_lsn = lsn; - if (checkpointState.newLastCheckpoint != null) { - await this.sourceRecordStore.postCommitCleanup(checkpointState.newLastCheckpoint, this.logger); + const cleanupCheckpoint = cleanupCheckpoints.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))[0]; + if (cleanupCheckpoint != null) { + await this.sourceRecordStore.postCommitCleanup(cleanupCheckpoint, this.logger); } } return { - checkpointBlocked: checkpointState.checkpointBlocked, - checkpointCreated: checkpointState.checkpointCreated + checkpointBlocked, + checkpointCreated }; } @@ -412,7 +427,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { await this.db.sync_rules.updateOne( { _id: this.group_id, - 'sync_configs._id': this.syncConfigId + 'sync_configs._id': { $in: this.syncConfigIds } }, { $set: { @@ -434,32 +449,34 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { const doc = await this.db.sync_rules.findOne( { _id: this.group_id, - 'sync_configs._id': this.syncConfigId + 'sync_configs._id': { $in: this.syncConfigIds } }, { session, projection: { state: 1, - sync_configs: { - $elemMatch: { - _id: this.syncConfigId - } - } + sync_configs: 1 } } ); - const state = (doc as ReplicationStreamDocumentV3)?.sync_configs?.[0]; + const states = + (doc as ReplicationStreamDocumentV3)?.sync_configs?.filter((config) => + this.syncConfigIds.some((id) => id.equals(config._id)) + ) ?? []; + if (doc == null || states.length == 0) { + return; + } + + const processingStates = states.filter((state) => state.state == storage.SyncRuleState.PROCESSING); if ( - doc && doc.state == storage.SyncRuleState.PROCESSING && - state?.state == storage.SyncRuleState.PROCESSING && - state.snapshot_done && - state.last_checkpoint != null + processingStates.length == states.length && + states.every((state) => state.snapshot_done && state.last_checkpoint != null) ) { await this.db.sync_rules.updateOne( { _id: this.group_id, - 'sync_configs._id': this.syncConfigId + 'sync_configs._id': { $in: this.syncConfigIds } }, { $set: { @@ -469,7 +486,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { }, { session, - arrayFilters: [{ 'config._id': this.syncConfigId }] + arrayFilters: [{ 'config._id': { $in: this.syncConfigIds } }] } ); @@ -482,7 +499,32 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { { session } ); activated = true; - } else if (doc?.state != storage.SyncRuleState.PROCESSING) { + } else if ( + doc.state == storage.SyncRuleState.ACTIVE && + processingStates.length > 0 && + processingStates.every((state) => state.snapshot_done && state.last_checkpoint != null) + ) { + await this.db.sync_rules.updateOne( + { + _id: this.group_id, + 'sync_configs._id': { $in: processingStates.map((state) => state._id) } + }, + { + $set: { + 'sync_configs.$[activeConfig].state': storage.SyncRuleState.STOP, + 'sync_configs.$[processingConfig].state': storage.SyncRuleState.ACTIVE + } + }, + { + session, + arrayFilters: [ + { 'activeConfig.state': storage.SyncRuleState.ACTIVE }, + { 'processingConfig._id': { $in: processingStates.map((state) => state._id) } } + ] + } + ); + activated = true; + } else if (doc.state != storage.SyncRuleState.PROCESSING && doc.state != storage.SyncRuleState.ACTIVE) { this.needsActivationV3 = false; } }); @@ -497,7 +539,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { await this.db.sync_rules.updateOne( { _id: this.group_id, - 'sync_configs._id': this.syncConfigId + 'sync_configs._id': { $in: this.syncConfigIds } }, { $set: { @@ -510,7 +552,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { }, { session: this.session, - arrayFilters: [{ 'config._id': this.syncConfigId }] + arrayFilters: [{ 'config._id': { $in: this.syncConfigIds } }] } ); } @@ -542,7 +584,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { await this.db.sync_rules.updateOne( { _id: this.group_id, - 'sync_configs._id': this.syncConfigId + 'sync_configs._id': { $in: this.syncConfigIds } }, { $set: { @@ -551,7 +593,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { }, { session: this.session, - arrayFilters: [{ 'config._id': this.syncConfigId }] + arrayFilters: [{ 'config._id': { $in: this.syncConfigIds } }] } ); } @@ -581,7 +623,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { await this.db.sync_rules.updateOne( { _id: this.group_id, - 'sync_configs._id': this.syncConfigId + 'sync_configs._id': { $in: this.syncConfigIds } }, { $set: { @@ -593,7 +635,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { }, { session: this.session, - arrayFilters: [{ 'config._id': this.syncConfigId }] + arrayFilters: [{ 'config._id': { $in: this.syncConfigIds } }] } ); } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index c81f0420b..8953eb4ca 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -26,8 +26,8 @@ import { MongoChecksums } from '../MongoChecksums.js'; import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; import { - MongoPersistedSyncRulesContentV1, - MongoPersistedSyncRulesContentV3 + MongoPersistedSyncConfigContentV1, + MongoPersistedSyncConfigContentV3 } from '../MongoPersistedSyncRulesContent.js'; import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js'; import { @@ -49,25 +49,25 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { declare readonly db: VersionedPowerSyncMongoV3; declare readonly checksums: MongoChecksumsV3; - private readonly syncRulesV3: MongoPersistedSyncRulesContentV3; + private readonly syncRulesV3: MongoPersistedSyncConfigContentV3; constructor( factory: MongoBucketStorage, group_id: number, - sync_rules: MongoPersistedSyncRulesContentV1, + sync_rules: MongoPersistedSyncConfigContentV1, slot_name: string, writeCheckpointMode: storage.WriteCheckpointMode | undefined, options: MongoSyncBucketStorageOptions ) { super(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options); - if (!(sync_rules instanceof MongoPersistedSyncRulesContentV3)) { + if (!(sync_rules instanceof MongoPersistedSyncConfigContentV3)) { throw new ServiceAssertionError('Missing sync config id for storage v3'); } this.syncRulesV3 = sync_rules; } - private get syncConfigId(): bson.ObjectId { - return this.syncRulesV3.syncConfigId; + private get syncConfigIds(): bson.ObjectId[] { + return this.syncRulesV3.syncConfigIds; } private get syncRulesCollection(): mongo.Collection { @@ -79,7 +79,7 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { _id: this.group_id, sync_configs: { $elemMatch: { - _id: this.syncConfigId, + _id: { $in: this.syncConfigIds }, ...extra } } @@ -89,27 +89,16 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { private syncConfigProjection(extra: mongo.Document = {}): mongo.Document { return { ...extra, - sync_configs: { - $elemMatch: { - _id: this.syncConfigId - } - } + sync_configs: 1 }; } private syncConfigArrayFilters(): mongo.UpdateOptions['arrayFilters'] { - return [{ 'config._id': this.syncConfigId }]; + return [{ 'config._id': { $in: this.syncConfigIds } }]; } - /** - * For now, we only support a single sync config per replication stream. - * - * In the future we'll add support for multiple. - */ - private selectedSyncConfig( - doc: Pick | null - ): SyncRuleConfigStateV3 | null { - return doc?.sync_configs?.[0] ?? null; + private selectedSyncConfigs(doc: Pick | null): SyncRuleConfigStateV3[] { + return doc?.sync_configs?.filter((config) => this.syncConfigIds.some((id) => id.equals(config._id))) ?? []; } protected async initializeVersionStorage(): Promise { @@ -174,13 +163,21 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { projection: this.syncConfigProjection() } ); - const syncConfig = this.selectedSyncConfig(doc); - if (!syncConfig?.snapshot_done) { + const syncConfigs = this.selectedSyncConfigs(doc); + if (syncConfigs.length == 0 || syncConfigs.some((config) => !config.snapshot_done)) { return null; } + const checkpoint = syncConfigs.reduce((min, config) => { + const value = config.last_checkpoint ?? 0n; + return min == null || value < min ? value : min; + }, null); return { - checkpoint: syncConfig.last_checkpoint ?? 0n, - lsn: syncConfig.last_checkpoint_lsn ?? null + checkpoint: checkpoint ?? 0n, + lsn: + syncConfigs + .map((config) => config.last_checkpoint_lsn) + .filter((lsn) => lsn != null) + .sort()[0] ?? null }; } @@ -188,13 +185,21 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { const doc = await this.syncRulesCollection.findOne(this.syncConfigMatch(), { projection: this.syncConfigProjection({ snapshot_lsn: 1 }) }); - const syncConfig = this.selectedSyncConfig(doc); - const checkpointLsn = syncConfig?.last_checkpoint_lsn ?? null; + const syncConfigs = this.selectedSyncConfigs(doc); + const checkpointLsn = + syncConfigs + .map((config) => config.last_checkpoint_lsn) + .filter((lsn) => lsn != null) + .sort()[0] ?? null; + const keepaliveOp = syncConfigs.reduce((min, config) => { + const value = config.keepalive_op ?? null; + return value != null && (min == null || value < min) ? value : min; + }, null); return { lastCheckpointLsn: checkpointLsn, resumeFromLsn: maxLsn(checkpointLsn, doc?.snapshot_lsn), - keepaliveOp: syncConfig?.keepalive_op ?? null, - syncConfigId: this.syncConfigId + keepaliveOp, + syncConfigIds: this.syncConfigIds }; } @@ -217,17 +222,29 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { const doc = await this.syncRulesCollection.findOne(this.syncConfigMatch(), { projection: this.syncConfigProjection({ state: 1, snapshot_lsn: 1, keepalive_op: 1 }) }); - const syncConfig = this.selectedSyncConfig(doc); - if (doc == null || syncConfig == null) { + const syncConfigs = this.selectedSyncConfigs(doc); + if (doc == null || syncConfigs.length == 0) { throw new ServiceAssertionError('Cannot find replication stream status'); } + const active = syncConfigs.some( + (config) => doc.state == storage.SyncRuleState.ACTIVE && config.state == storage.SyncRuleState.ACTIVE + ); + const checkpointLsn = + syncConfigs + .map((config) => config.last_checkpoint_lsn) + .filter((lsn) => lsn != null) + .sort()[0] ?? null; + const keepaliveOp = syncConfigs.reduce((min, config) => { + const value = config.keepalive_op ?? null; + return value != null && (min == null || value < min) ? value : min; + }, null); return { - snapshot_done: syncConfig.snapshot_done ?? false, + snapshot_done: syncConfigs.every((config) => config.snapshot_done ?? false), snapshot_lsn: doc.snapshot_lsn ?? null, - active: doc.state == storage.SyncRuleState.ACTIVE && syncConfig.state == storage.SyncRuleState.ACTIVE, - checkpoint_lsn: syncConfig.last_checkpoint_lsn ?? null, - keepalive_op: syncConfig.keepalive_op ?? null + active, + checkpoint_lsn: checkpointLsn, + keepalive_op: keepaliveOp }; } diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index 4011dac5a..0cdcf2ece 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -57,9 +57,10 @@ bucket_definitions: `) ); const bucketStorage = factory.getInstance(syncRules); + const syncRulesContent = (await factory.getReplicationStreamConfigs(syncRules.id))[0]; const { checkpoint } = await populate(bucketStorage, 1); - return { bucketStorage, checkpoint, factory, syncRules }; + return { bucketStorage, checkpoint, factory, syncRules: syncRulesContent }; }; test('full compact', async () => { @@ -115,6 +116,7 @@ bucket_definitions: `) ); const bucketStorage = factory.getInstance(syncRules); + const syncRulesContent = (await factory.getReplicationStreamConfigs(syncRules.id))[0]; const storageDb = (bucketStorage as any).db; await populate(bucketStorage, 2); @@ -141,7 +143,7 @@ bucket_definitions: expect(result2.buckets).toEqual(0); const users = ['u1', 'u2']; - const userRequests = users.map((user) => bucketRequest(syncRules, `by_user2["${user}"]`)); + const userRequests = users.map((user) => bucketRequest(syncRulesContent, `by_user2["${user}"]`)); const [u1Request, u2Request] = userRequests; const checksumAfter = await bucketStorage.getChecksums(checkpoint, userRequests); expect(checksumAfter.get(u1Request.bucket)).toEqual({ diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index f1265f460..26a0ee0a1 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -1,10 +1,17 @@ import { deserializeParameterLookup, JwtPayload, storage, updateSyncRulesFromYaml } from '@powersync/service-core'; import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; -import { DEFAULT_HYDRATION_STATE, nodeSqlite, RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules'; +import { + DEFAULT_HYDRATION_STATE, + nodeSqlite, + RequestParameters, + ScopedParameterLookup, + SqlSyncRules +} from '@powersync/service-sync-rules'; import * as bson from 'bson'; import * as sqlite from 'node:sqlite'; import { describe, expect, test } from 'vitest'; import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js'; +import { MongoPersistedReplicationStream } from '../../src/storage/implementation/MongoPersistedSyncRulesContent.js'; import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js'; import { SourceRecordStoreV3 } from '../../src/storage/implementation/v3/SourceRecordStoreV3.js'; import type { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js'; @@ -82,6 +89,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor ) ); const bucketStorage = factory.getInstance(syncRules); + const syncRulesContent = (await factory.getReplicationStreamConfigs(syncRules.id))[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -136,7 +144,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor const options: storage.BucketDataBatchOptions = {}; const batch1 = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)], options) + bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRulesContent, 'global[]', 0n)], options) ); expect(test_utils.getBatchData(batch1)).toEqual([ { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }, @@ -151,7 +159,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor const batch2 = await test_utils.fromAsync( bucketStorage.getBucketDataBatch( checkpoint, - [bucketRequest(syncRules, 'global[]', batch1[0].chunkData.next_after)], + [bucketRequest(syncRulesContent, 'global[]', batch1[0].chunkData.next_after)], options ) ); @@ -167,7 +175,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor const batch3 = await test_utils.fromAsync( bucketStorage.getBucketDataBatch( checkpoint, - [bucketRequest(syncRules, 'global[]', batch2[0].chunkData.next_after)], + [bucketRequest(syncRulesContent, 'global[]', batch2[0].chunkData.next_after)], options ) ); @@ -531,7 +539,8 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor ) ); const bucketStorage = factory.getInstance(syncRules); - const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig(); + const syncRulesContent = (await factory.getReplicationStreamConfigs(syncRules.id))[0]; + const sync_rules = syncRulesContent.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); @@ -552,7 +561,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor const parameters = new RequestParameters(new JwtPayload({ sub: 'u1', parameters: { test: 'shape-check' } }), {}); const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier; const buckets = await querier.queryDynamicBucketDescriptions({ - async getParameterSets(lookups) { + async getParameterSets(lookups: ScopedParameterLookup[]) { expect(lookups.map((l) => l.indexKey)).toEqual([['shape-check']]); expect(lookups[0].indexId).toEqual('1'); @@ -561,7 +570,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor return parameter_sets; } }); - expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(syncRules, 'global["user-1"]').bucket]); + expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(syncRulesContent, 'global["user-1"]').bucket]); const mongoFactory = factory as MongoBucketStorage; const db = (bucketStorage as MongoSyncBucketStorage).db as VersionedPowerSyncMongoV3; @@ -588,6 +597,218 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor expect(deserializeParameterLookup(parameterEntry!.lookup)).toEqual(['shape-check']); }); + test.runIf(storageVersion >= 3)('replaces an existing deploying sync config', async () => { + await using factory = await storageConfig.factory(); + const mongoFactory = factory as MongoBucketStorage; + + const first = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` +config: + edition: 2 + +streams: + by_owner: + query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id') +`, + { storageVersion } + ) + ); + const second = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` +config: + edition: 2 + +streams: + by_project: + query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id') +`, + { storageVersion } + ) + ); + + expect(second.id).not.toEqual(first.id); + expect((await mongoFactory.db.sync_rules.findOne({ _id: first.id }))?.state).toBe(storage.SyncRuleState.STOP); + + const replicatingStreams = await factory.getReplicatingReplicationStreams(); + expect(replicatingStreams).toHaveLength(1); + expect(replicatingStreams[0].id).toEqual(second.id); + + const stream = (await mongoFactory.db.sync_rules.findOne({ _id: second.id })) as ReplicationStreamDocumentV3; + expect(stream.sync_configs).toHaveLength(1); + expect(stream.sync_configs[0].state).toBe(storage.SyncRuleState.PROCESSING); + + const configs = await factory.getReplicationStreamConfigs(second.id); + expect(configs).toHaveLength(1); + const statuses = await factory.getReplicationStreamConfigStatuses(second.id); + expect(statuses.map((status) => status.id).sort()).toEqual( + stream.sync_configs.map((config) => config._id.toHexString()).sort() + ); + const parsed = (replicatingStreams[0] as MongoPersistedReplicationStream) + .toSyncConfigContent() + .parsed(test_utils.PARSE_OPTIONS); + expect(parsed.syncConfigs).toHaveLength(1); + expect(parsed.hydratedSyncConfig().bucketDataSources).toHaveLength(1); + + const bucketStorage = mongoFactory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorage; + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTableId = new bson.ObjectId('6544e3899293153fa7b3834b'); + const resolved = await writer.resolveTables({ + connection_id: 1, + source: sourceDescriptor('todos', { objectId: 'todos-relation' }), + idGenerator: objectIdGenerator(sourceTableId.toHexString()) + }); + expect(resolved.tables).toHaveLength(1); + expect(resolved.tables[0].bucketDataSources).toHaveLength(1); + + const sourceTable = await (bucketStorage.db as VersionedPowerSyncMongoV3) + .sourceTablesV3(second.id) + .findOne({ _id: sourceTableId }); + expect(sourceTable?.bucket_data_source_ids).toHaveLength(1); + }); + + test.runIf(storageVersion >= 3)('removing one config keeps shared source-table membership', async () => { + await using factory = await storageConfig.factory(); + const mongoFactory = factory as MongoBucketStorage; + + const first = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` +config: + edition: 3 + +streams: + by_owner: + query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id') +`, + { storageVersion } + ) + ); + const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage; + await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); + await firstWriter.markAllSnapshotDone('1/1'); + await firstWriter.commit('1/1'); + + const second = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` +config: + edition: 3 + +streams: + by_project: + query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id') +`, + { storageVersion } + ) + ); + expect(second.id).toEqual(first.id); + + const bucketStorage = mongoFactory.getInstance(second) as MongoSyncBucketStorage; + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const sourceTableId = new bson.ObjectId('6544e3899293153fa7b3834c'); + const source = sourceDescriptor('todos', { objectId: 'todos-relation' }); + + const resolved = await writer.resolveTables({ + connection_id: 1, + source, + idGenerator: objectIdGenerator(sourceTableId.toHexString()) + }); + expect(resolved.tables).toHaveLength(1); + + const db = bucketStorage.db as VersionedPowerSyncMongoV3; + const before = await db.sourceTablesV3(first.id).findOne({ _id: sourceTableId }); + expect(before?.bucket_data_source_ids).toHaveLength(2); + + await writer.markAllSnapshotDone('2/1'); + await writer.commit('2/1'); + + const activeStorage = (await factory.getActiveStorage()) as MongoSyncBucketStorage; + await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS); + const activeStatus = await activeWriter.getSourceTableStatus(resolved.tables[0]); + expect(activeStatus?.bucketDataSources).toHaveLength(1); + }); + + test.runIf(storageVersion >= 3)( + 'keeps compatible active and deploying sync configs in one replication stream', + async () => { + await using factory = await storageConfig.factory(); + const mongoFactory = factory as MongoBucketStorage; + + const first = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` +config: + edition: 3 + +streams: + by_owner: + query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id') +`, + { storageVersion } + ) + ); + const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage; + await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); + await firstWriter.markAllSnapshotDone('1/1'); + await firstWriter.commit('1/1'); + + let configs = await factory.getReplicationStreamConfigs(first.id); + const firstConfigId = configs[0].syncConfigId; + expect((await factory.getActiveSyncConfigContent())?.syncConfigId).toBe(firstConfigId); + + const second = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` +config: + edition: 3 + +streams: + by_project: + query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id') +`, + { storageVersion } + ) + ); + expect(second.id).toEqual(first.id); + + configs = await factory.getReplicationStreamConfigs(first.id); + const deploying = await factory.getDeployingSyncConfigContent(); + expect(configs).toHaveLength(2); + expect(deploying).not.toBeNull(); + expect((await factory.getActiveSyncConfigContent())?.syncConfigId).toBe(firstConfigId); + + const stream = (await mongoFactory.db.sync_rules.findOne({ _id: first.id })) as ReplicationStreamDocumentV3; + expect(stream.state).toBe(storage.SyncRuleState.ACTIVE); + expect(stream.sync_configs.map((config) => config.state).sort()).toEqual([ + storage.SyncRuleState.ACTIVE, + storage.SyncRuleState.PROCESSING + ]); + + const replicatingStreams = await factory.getReplicatingReplicationStreams(); + expect(replicatingStreams).toHaveLength(1); + expect(replicatingStreams[0].replicationJobId).toContain(stream.sync_configs[0]._id.toHexString()); + expect(replicatingStreams[0].replicationJobId).toContain(stream.sync_configs[1]._id.toHexString()); + + const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorage; + await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS); + await secondWriter.markAllSnapshotDone('2/1'); + await secondWriter.commit('2/1'); + + const updatedStream = (await mongoFactory.db.sync_rules.findOne({ + _id: first.id + })) as ReplicationStreamDocumentV3; + expect(updatedStream.state).toBe(storage.SyncRuleState.ACTIVE); + expect(updatedStream.sync_configs.map((config) => config.state).sort()).toEqual([ + storage.SyncRuleState.ACTIVE, + storage.SyncRuleState.STOP + ]); + expect(await factory.getDeployingSyncConfigContent()).toBeNull(); + expect((await factory.getActiveSyncConfigContent())?.syncConfigId).not.toBe(firstConfigId); + } + ); + test.runIf(storageVersion < 3)('can replace processing legacy sync rules', async () => { await using factory = await storageConfig.factory(); diff --git a/modules/module-mongodb-storage/test/src/storeCurrentData.test.ts b/modules/module-mongodb-storage/test/src/storeCurrentData.test.ts index b0a8d4ab1..995e4efe4 100644 --- a/modules/module-mongodb-storage/test/src/storeCurrentData.test.ts +++ b/modules/module-mongodb-storage/test/src/storeCurrentData.test.ts @@ -81,6 +81,7 @@ function registerStoreCurrentDataTests(storageVersion: number) { await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion })); const bucketStorage = factory.getInstance(syncRules); + const syncRulesContent = (await factory.getReplicationStreamConfigs(syncRules.id))[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -114,6 +115,7 @@ function registerStoreCurrentDataTests(storageVersion: number) { await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion })); const bucketStorage = factory.getInstance(syncRules); + const syncRulesContent = (await factory.getReplicationStreamConfigs(syncRules.id))[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const table = await test_utils.resolveTestTable(writer, 'test_data', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); @@ -129,7 +131,7 @@ function registerStoreCurrentDataTests(storageVersion: number) { const checkpoint = flushResult!.flushed_op; const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)]) + bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRulesContent, 'global[]', 0n)]) ); expect(test_utils.getBatchData(batch)).toMatchObject([{ op: 'PUT', object_id: 'test1' }]); @@ -141,6 +143,7 @@ function registerStoreCurrentDataTests(storageVersion: number) { await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion })); const bucketStorage = factory.getInstance(syncRules); + const syncRulesContent = (await factory.getReplicationStreamConfigs(syncRules.id))[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const table = await test_utils.resolveTestTable(writer, 'test_data', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); @@ -156,7 +159,7 @@ function registerStoreCurrentDataTests(storageVersion: number) { const checkpoint = flushResult!.flushed_op; const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)]) + bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRulesContent, 'global[]', 0n)]) ); expect(test_utils.getBatchData(batch)).toMatchObject([{ op: 'PUT', object_id: 'test1' }]); @@ -169,6 +172,7 @@ function registerStoreCurrentDataTests(storageVersion: number) { await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion })); const bucketStorage = factory.getInstance(syncRules); + const syncRulesContent = (await factory.getReplicationStreamConfigs(syncRules.id))[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const table = await test_utils.resolveTestTable(writer, 'test_data', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); @@ -192,7 +196,7 @@ function registerStoreCurrentDataTests(storageVersion: number) { const checkpoint = flushResult!.flushed_op; const batch = await test_utils.fromAsync( - bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)]) + bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRulesContent, 'global[]', 0n)]) ); const data = test_utils.getBatchData(batch); expect(data.length).toBeGreaterThan(0); diff --git a/modules/module-mongodb/test/src/change_stream.test.ts b/modules/module-mongodb/test/src/change_stream.test.ts index 6b286d233..f1d31b1ff 100644 --- a/modules/module-mongodb/test/src/change_stream.test.ts +++ b/modules/module-mongodb/test/src/change_stream.test.ts @@ -820,7 +820,7 @@ bucket_definitions: // Simulate an error await context.storage!.reportError(new Error('simulated error')); - const syncRules = await context.factory.getActiveSyncRulesContent(); + const syncRules = await context.factory.getActiveSyncConfigContent(); expect(syncRules).toBeTruthy(); expect(syncRules?.last_fatal_error).toEqual('simulated error'); @@ -830,7 +830,7 @@ bucket_definitions: // Just wait, and check that the error is cleared automatically. await vi.waitUntil( async () => { - const error = (await context.factory.getActiveSyncRulesContent())?.last_fatal_error; + const error = (await context.factory.getActiveSyncConfigContent())?.last_fatal_error; return error == null; }, { timeout: 2_000 } diff --git a/modules/module-mongodb/test/src/change_stream_utils.ts b/modules/module-mongodb/test/src/change_stream_utils.ts index 9d69aaac0..4f1a356ed 100644 --- a/modules/module-mongodb/test/src/change_stream_utils.ts +++ b/modules/module-mongodb/test/src/change_stream_utils.ts @@ -30,7 +30,7 @@ export class ChangeStreamTestContext { private _walStream?: ChangeStream; private abortController = new AbortController(); private settledReplicationPromise?: Promise>; - private syncRulesContent?: storage.PersistedSyncRulesContent; + private syncRulesContent?: storage.PersistedSyncConfigContent; public storage?: SyncRulesBucketStorage; /** @@ -101,26 +101,30 @@ export class ChangeStreamTestContext { } async updateSyncRules(content: string) { - const syncRules = await this.factory.updateSyncRules( + const replicationStream = await this.factory.updateSyncRules( updateSyncRulesFromYaml(content, { validate: true, storageVersion: this.storageVersion }) ); - this.syncRulesContent = syncRules; - this.storage = this.factory.getInstance(syncRules); + this.syncRulesContent = (await this.factory.getReplicationStreamConfigs(replicationStream.id))[0]; + this.storage = this.factory.getInstance(replicationStream); return this.storage!; } async loadNextSyncRules() { - const syncRules = await this.factory.getNextSyncRulesContent(); + const syncRules = await this.factory.getDeployingSyncConfigContent(); if (syncRules == null) { throw new Error(`Next sync config not available`); } this.syncRulesContent = syncRules; - this.storage = this.factory.getInstance(syncRules); + const replicationStream = await this.factory.getReplicationStream(syncRules.replicationStreamId); + if (replicationStream == null) { + throw new Error(`Next replication stream not available`); + } + this.storage = this.factory.getInstance(replicationStream); return this.storage!; } - private getSyncRulesContent(): storage.PersistedSyncRulesContent { + private getSyncRulesContent(): storage.PersistedSyncConfigContent { if (this.syncRulesContent == null) { throw new Error('Sync config not configured - call updateSyncRules() first'); } diff --git a/modules/module-mongodb/test/src/resume.test.ts b/modules/module-mongodb/test/src/resume.test.ts index d014e67d2..a4eda9d17 100644 --- a/modules/module-mongodb/test/src/resume.test.ts +++ b/modules/module-mongodb/test/src/resume.test.ts @@ -67,8 +67,10 @@ function defineResumeTest({ factory: factoryGenerator, storageVersion }: Storage // Create a new context without updating the sync config await using context2 = new ChangeStreamTestContext(factory, connectionManager, {}, storageVersion); - const activeContent = await factory.getActiveSyncRulesContent(); - context2.storage = factory.getInstance(activeContent!); + const activeContent = await factory.getActiveSyncConfigContent(); + const activeStream = + activeContent == null ? null : await factory.getReplicationStream(activeContent.replicationStreamId); + context2.storage = factory.getInstance(activeStream!); // If this test times out, it likely didn't throw the expected error here. const result = await context2.startStreaming(); diff --git a/modules/module-mssql/test/src/CDCStreamTestContext.ts b/modules/module-mssql/test/src/CDCStreamTestContext.ts index d3eb8410b..b5e06f799 100644 --- a/modules/module-mssql/test/src/CDCStreamTestContext.ts +++ b/modules/module-mssql/test/src/CDCStreamTestContext.ts @@ -26,7 +26,7 @@ export class CDCStreamTestContext implements AsyncDisposable { private _cdcStream?: CDCStream; private abortController = new AbortController(); private streamPromise?: Promise; - private syncRulesContent?: storage.PersistedSyncRulesContent; + private syncRulesContent?: storage.PersistedSyncConfigContent; public storage?: SyncRulesBucketStorage; private snapshotPromise?: Promise; private replicationDone = false; @@ -75,37 +75,45 @@ export class CDCStreamTestContext implements AsyncDisposable { } async updateSyncRules(content: string) { - const syncRules = await this.factory.updateSyncRules( + const replicationStream = await this.factory.updateSyncRules( updateSyncRulesFromYaml(content, { validate: true, storageVersion: LEGACY_STORAGE_VERSION }) ); - this.syncRulesContent = syncRules; - this.storage = this.factory.getInstance(syncRules); + this.syncRulesContent = (await this.factory.getReplicationStreamConfigs(replicationStream.id))[0]; + this.storage = this.factory.getInstance(replicationStream); return this.storage!; } async loadNextSyncRules() { - const syncRules = await this.factory.getNextSyncRulesContent(); + const syncRules = await this.factory.getDeployingSyncConfigContent(); if (syncRules == null) { throw new Error(`Next replication stream not available`); } this.syncRulesContent = syncRules; - this.storage = this.factory.getInstance(syncRules); + const replicationStream = await this.factory.getReplicationStream(syncRules.replicationStreamId); + if (replicationStream == null) { + throw new Error(`Next replication stream not available`); + } + this.storage = this.factory.getInstance(replicationStream); return this.storage!; } async loadActiveSyncRules() { - const syncRules = await this.factory.getActiveSyncRulesContent(); + const syncRules = await this.factory.getActiveSyncConfigContent(); if (syncRules == null) { throw new Error(`Active replication stream not available`); } this.syncRulesContent = syncRules; - this.storage = this.factory.getInstance(syncRules); + const replicationStream = await this.factory.getReplicationStream(syncRules.replicationStreamId); + if (replicationStream == null) { + throw new Error(`Active replication stream not available`); + } + this.storage = this.factory.getInstance(replicationStream); return this.storage!; } - private getSyncRulesContent(): storage.PersistedSyncRulesContent { + private getSyncRulesContent(): storage.PersistedSyncConfigContent { if (this.syncRulesContent == null) { throw new Error('Sync config not configured - call updateSyncRules() first'); } diff --git a/modules/module-mysql/test/src/BinlogStreamUtils.ts b/modules/module-mysql/test/src/BinlogStreamUtils.ts index c2d120d22..f99f242ad 100644 --- a/modules/module-mysql/test/src/BinlogStreamUtils.ts +++ b/modules/module-mysql/test/src/BinlogStreamUtils.ts @@ -33,7 +33,7 @@ export class BinlogStreamTestContext { private streamPromise?: Promise; public storage?: SyncRulesBucketStorage; private replicationDone = false; - private syncRulesContent?: storage.PersistedSyncRulesContent; + private syncRulesContent?: storage.PersistedSyncConfigContent; static async open(factory: storage.TestStorageFactory, options?: { doNotClear?: boolean }) { const f = await factory({ doNotClear: options?.doNotClear }); @@ -71,38 +71,46 @@ export class BinlogStreamTestContext { } async updateSyncRules(content: string): Promise { - const syncRules = await this.factory.updateSyncRules( + const replicationStream = await this.factory.updateSyncRules( updateSyncRulesFromYaml(content, { validate: true, storageVersion: LEGACY_STORAGE_VERSION }) ); - this.syncRulesContent = syncRules; - this.storage = this.factory.getInstance(syncRules); + this.syncRulesContent = (await this.factory.getReplicationStreamConfigs(replicationStream.id))[0]; + this.storage = this.factory.getInstance(replicationStream); return this.storage!; } async loadNextSyncRules() { - const syncRules = await this.factory.getNextSyncRulesContent(); + const syncRules = await this.factory.getDeployingSyncConfigContent(); if (syncRules == null) { throw new Error(`Next replication stream not available`); } this.syncRulesContent = syncRules; - this.storage = this.factory.getInstance(syncRules); + const replicationStream = await this.factory.getReplicationStream(syncRules.replicationStreamId); + if (replicationStream == null) { + throw new Error(`Next replication stream not available`); + } + this.storage = this.factory.getInstance(replicationStream); return this.storage!; } async loadActiveSyncRules() { - const syncRules = await this.factory.getActiveSyncRulesContent(); + const syncRules = await this.factory.getActiveSyncConfigContent(); if (syncRules == null) { throw new Error(`Active replication stream not available`); } this.syncRulesContent = syncRules; - this.storage = this.factory.getInstance(syncRules); + const replicationStream = await this.factory.getReplicationStream(syncRules.replicationStreamId); + if (replicationStream == null) { + throw new Error(`Active replication stream not available`); + } + this.storage = this.factory.getInstance(replicationStream); this.replicationDone = true; return this.storage!; } - private getSyncRulesContent(): storage.PersistedSyncRulesContent { + private getSyncRulesContent(): storage.PersistedSyncConfigContent { if (this.syncRulesContent == null) { throw new Error('Sync config not configured - call updateSyncRules() first'); } diff --git a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts index 799fbdf6d..7f02d51b8 100644 --- a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts @@ -10,7 +10,10 @@ import { getStorageApplicationName } from '../utils/application-name.js'; import { NOTIFICATION_CHANNEL, STORAGE_SCHEMA_NAME } from '../utils/db.js'; import { notifySyncRulesUpdate } from './batch/PostgresBucketBatch.js'; import { PostgresSyncRulesStorage } from './PostgresSyncRulesStorage.js'; -import { PostgresPersistedSyncRulesContent } from './sync-rules/PostgresPersistedSyncRulesContent.js'; +import { + PostgresPersistedReplicationStream, + PostgresPersistedSyncRulesContent +} from './sync-rules/PostgresPersistedSyncRulesContent.js'; export type PostgresBucketStorageOptions = { config: NormalizedPostgresStorageConfig; @@ -49,26 +52,30 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory { } getInstance( - syncRules: storage.PersistedSyncRulesContent, + replicationStream: storage.PersistedReplicationStream, options?: GetIntanceOptions ): storage.SyncRulesBucketStorage { - const storage = new PostgresSyncRulesStorage({ + const syncRulesContent = + replicationStream instanceof PostgresPersistedReplicationStream + ? replicationStream.toSyncConfigContent() + : replicationStream; + const syncRuleStorage = new PostgresSyncRulesStorage({ factory: this, db: this.db, - sync_rules: syncRules, + sync_rules: syncRulesContent as storage.PersistedSyncConfigContent, batchLimits: this.options.config.batch_limits }); if (!options?.skipLifecycleHooks) { - this.iterateListeners((cb) => cb.syncStorageCreated?.(storage)); + this.iterateListeners((cb) => cb.syncStorageCreated?.(syncRuleStorage)); } - storage.registerListener({ + syncRuleStorage.registerListener({ batchStarted: (batch) => { batch.registerListener({ replicationEvent: (payload) => this.iterateListeners((cb) => cb.replicationEvent?.(payload)) }); } }); - return storage; + return syncRuleStorage; } async getStorageMetrics(): Promise { @@ -153,7 +160,7 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory { }; } - async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise { + async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise { const storageVersion = options.storageVersion ?? options.config.parsed.config.storageVersion ?? storage.CURRENT_STORAGE_VERSION; const storageConfig = storage.STORAGE_VERSION_CONFIG[storageVersion]; @@ -220,7 +227,7 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory { await notifySyncRulesUpdate(this.db, newSyncRulesRow!); - return new PostgresPersistedSyncRulesContent(this.db, newSyncRulesRow!); + return new PostgresPersistedReplicationStream(this.db, newSyncRulesRow!); }); } @@ -253,8 +260,8 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory { } async restartReplication(sync_rules_group_id: number): Promise { - const next = await this.getNextSyncRulesContent(); - const active = await this.getActiveSyncRulesContent(); + const next = await this.getDeployingSyncConfigContent(); + const active = await this.getActiveSyncConfigContent(); // In both the below cases, we create a new replication stream. // The current one will continue serving sync requests until the next one has finished processing. @@ -299,7 +306,7 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory { } } - async getActiveSyncRulesContent(): Promise { + async getActiveSyncConfigContent(): Promise { const activeRow = await this.db.sql` SELECT * @@ -322,8 +329,17 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory { return new PostgresPersistedSyncRulesContent(this.db, activeRow); } - async getNextSyncRulesContent(): Promise { - const nextRow = await this.db.sql` + async getActiveSyncConfigStatus(): Promise { + const content = await this.getActiveSyncConfigContent(); + if (content == null) { + return null; + } + + return content.getSyncConfigStatus(); + } + + async getDeployingSyncConfigContent(): Promise { + const row = await this.db.sql` SELECT * FROM @@ -337,14 +353,71 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory { ` .decoded(models.SyncRules) .first(); - if (!nextRow) { + + return row == null ? null : new PostgresPersistedSyncRulesContent(this.db, row); + } + + async getReplicationStreamConfigs(replicationStreamId: number): Promise { + const row = await this.db.sql` + SELECT + * + FROM + sync_rules + WHERE + id = ${{ value: replicationStreamId, type: 'int4' }} + ` + .decoded(models.SyncRules) + .first(); + if (row == null) { + return []; + } + + return [new PostgresPersistedSyncRulesContent(this.db, row)]; + } + + async getSyncConfigContent( + syncConfigId: storage.PersistedSyncConfigId + ): Promise { + const replicationStreamId = Number(syncConfigId); + if (!Number.isInteger(replicationStreamId)) { return null; } - return new PostgresPersistedSyncRulesContent(this.db, nextRow); + const row = await this.db.sql` + SELECT + * + FROM + sync_rules + WHERE + id = ${{ value: replicationStreamId, type: 'int4' }} + ` + .decoded(models.SyncRules) + .first(); + if (row == null) { + return null; + } + + return new PostgresPersistedSyncRulesContent(this.db, row); } - async getReplicatingSyncRules(): Promise { + async getReplicationStream(replicationStreamId: number): Promise { + const row = await this.db.sql` + SELECT + * + FROM + sync_rules + WHERE + id = ${{ value: replicationStreamId, type: 'int4' }} + ` + .decoded(models.SyncRules) + .first(); + if (row == null) { + return null; + } + return new PostgresPersistedReplicationStream(this.db, row); + } + + async getReplicatingReplicationStreams(): Promise { const rows = await this.db.sql` SELECT * @@ -357,10 +430,10 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory { .decoded(models.SyncRules) .rows(); - return rows.map((row) => new PostgresPersistedSyncRulesContent(this.db, row)); + return rows.map((row) => new PostgresPersistedReplicationStream(this.db, row)); } - async getStoppedSyncRules(): Promise { + async getStoppedReplicationStreams(): Promise { const rows = await this.db.sql` SELECT * @@ -372,22 +445,37 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory { .decoded(models.SyncRules) .rows(); - return rows.map((row) => new PostgresPersistedSyncRulesContent(this.db, row)); + return rows.map((row) => new PostgresPersistedReplicationStream(this.db, row)); } async getActiveStorage(): Promise { - const content = await this.getActiveSyncRulesContent(); - if (content == null) { + const activeRow = await this.db.sql` + SELECT + * + FROM + sync_rules + WHERE + state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }} + OR state = ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }} + ORDER BY + id DESC + LIMIT + 1 + ` + .decoded(models.SyncRules) + .first(); + if (!activeRow) { return null; } + const stream = new PostgresPersistedReplicationStream(this.db, activeRow); // It is important that this instance is cached. // Not for the instance construction itself, but to ensure that internal caches on the instance // are re-used properly. - if (this.activeStorageCache?.group_id == content.id) { + if (this.activeStorageCache?.group_id == stream.id) { return this.activeStorageCache; } else { - const instance = this.getInstance(content); + const instance = this.getInstance(stream); this.activeStorageCache = instance; return instance; } diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index af36fea7b..751adb04b 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -41,7 +41,7 @@ import { PostgresCompactor } from './PostgresCompactor.js'; export type PostgresSyncRulesStorageOptions = { factory: PostgresBucketStorageFactory; db: lib_postgres.DatabaseClient; - sync_rules: storage.PersistedSyncRulesContent; + sync_rules: storage.PersistedSyncConfigContent; write_checkpoint_mode?: storage.WriteCheckpointMode; batchLimits: RequiredOperationBatchLimits; }; @@ -53,7 +53,7 @@ export class PostgresSyncRulesStorage [framework.DO_NOT_LOG] = true; public readonly group_id: number; - public readonly sync_rules: storage.PersistedSyncRulesContent; + public readonly sync_rules: storage.PersistedSyncConfigContent; public readonly slot_name: string; public readonly factory: PostgresBucketStorageFactory; public readonly storageConfig: StorageVersionConfig; diff --git a/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts b/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts index 6d2ae37f3..6720bfb74 100644 --- a/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts +++ b/modules/module-postgres-storage/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts @@ -3,9 +3,7 @@ import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; import { storage } from '@powersync/service-core'; import { models } from '../../types/types.js'; -export class PostgresPersistedSyncRulesContent extends storage.PersistedSyncRulesContent { - current_lock: storage.ReplicationLock | null = null; - +export class PostgresPersistedSyncRulesContent extends storage.PersistedSyncConfigContent { constructor( private db: lib_postgres.DatabaseClient, row: models.SyncRulesDecoded @@ -20,9 +18,30 @@ export class PostgresPersistedSyncRulesContent extends storage.PersistedSyncRule last_checkpoint_ts: row.last_checkpoint_ts ? new Date(row.last_checkpoint_ts) : null, last_keepalive_ts: row.last_keepalive_ts ? new Date(row.last_keepalive_ts) : null, active: row.state == 'ACTIVE', + state: row.state as storage.SyncRuleState, storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION }); } +} + +export class PostgresPersistedReplicationStream extends storage.PersistedReplicationStream { + current_lock: storage.ReplicationLock | null = null; + + constructor( + private db: lib_postgres.DatabaseClient, + private readonly row: models.SyncRulesDecoded + ) { + super({ + id: Number(row.id), + slot_name: row.slot_name, + state: row.state as storage.SyncRuleState, + storageVersion: row.storage_version ?? storage.LEGACY_STORAGE_VERSION + }); + } + + toSyncConfigContent(): PostgresPersistedSyncRulesContent { + return new PostgresPersistedSyncRulesContent(this.db, this.row); + } async lock(): Promise { const manager = new lib_postgres.PostgresLockManager({ diff --git a/modules/module-postgres-storage/test/src/storage.test.ts b/modules/module-postgres-storage/test/src/storage.test.ts index 446592614..73f4512e5 100644 --- a/modules/module-postgres-storage/test/src/storage.test.ts +++ b/modules/module-postgres-storage/test/src/storage.test.ts @@ -39,7 +39,8 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { ) ); const bucketStorage = factory.getInstance(syncRules); - const globalBucket = bucketRequest(syncRules, 'global[]'); + const syncRulesContent = (await factory.getReplicationStreamConfigs(syncRules.id))[0]; + const globalBucket = bucketRequest(syncRulesContent, 'global[]'); const result = await (async () => { await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); diff --git a/modules/module-postgres-storage/test/src/storage_compacting.test.ts b/modules/module-postgres-storage/test/src/storage_compacting.test.ts index 392750bdc..decea348f 100644 --- a/modules/module-postgres-storage/test/src/storage_compacting.test.ts +++ b/modules/module-postgres-storage/test/src/storage_compacting.test.ts @@ -17,6 +17,7 @@ bucket_definitions: `) ); const bucketStorage = factory.getInstance(syncRules); + const syncRulesContent = (await factory.getReplicationStreamConfigs(syncRules.id))[0]; const result = await (async () => { await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -44,12 +45,12 @@ bucket_definitions: // Compact with an explicit bucket name — exercises the this.buckets // iteration path, NOT the compactAllBuckets discovery path. await bucketStorage.compact({ - compactBuckets: [bucketRequest(syncRules, 'global[]').bucket], + compactBuckets: [bucketRequest(syncRulesContent, 'global[]').bucket], minBucketChanges: 1 }); const batch = await test_utils.oneFromAsync( - bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)]) + bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRulesContent, 'global[]', 0n)]) ); expect(batch.chunkData.data).toMatchObject([ @@ -70,7 +71,8 @@ bucket_definitions: `) ); const bucketStorage = factory.getInstance(syncRules); - const request = bucketRequest(syncRules, 'global[]'); + const syncRulesContent = (await factory.getReplicationStreamConfigs(syncRules.id))[0]; + const request = bucketRequest(syncRulesContent, 'global[]'); const result = await (async () => { await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); diff --git a/modules/module-postgres-storage/test/src/storage_sync.test.ts b/modules/module-postgres-storage/test/src/storage_sync.test.ts index 88965e59b..0361331e2 100644 --- a/modules/module-postgres-storage/test/src/storage_sync.test.ts +++ b/modules/module-postgres-storage/test/src/storage_sync.test.ts @@ -34,7 +34,8 @@ function registerStorageVersionTests(storageVersion: number) { ) ); const bucketStorage = factory.getInstance(syncRules); - const globalBucket = bucketRequest(syncRules, 'global[]'); + const syncRulesContent = (await factory.getReplicationStreamConfigs(syncRules.id))[0]; + const globalBucket = bucketRequest(syncRulesContent, 'global[]'); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); diff --git a/modules/module-postgres/src/module/PostgresModule.ts b/modules/module-postgres/src/module/PostgresModule.ts index d762110c0..47a3819e9 100644 --- a/modules/module-postgres/src/module/PostgresModule.ts +++ b/modules/module-postgres/src/module/PostgresModule.ts @@ -79,14 +79,15 @@ export class PostgresModule extends replication.ReplicationModule>; @@ -89,37 +89,45 @@ export class WalStreamTestContext implements AsyncDisposable { } async updateSyncRules(content: string) { - const syncRules = await this.factory.updateSyncRules( + const replicationStream = await this.factory.updateSyncRules( updateSyncRulesFromYaml(content, { validate: true, storageVersion: this.storageVersion }) ); - this.syncRulesContent = syncRules; - this.storage = this.factory.getInstance(syncRules); + this.syncRulesContent = (await this.factory.getReplicationStreamConfigs(replicationStream.id))[0]; + this.storage = this.factory.getInstance(replicationStream); return this.storage!; } async loadNextSyncRules() { - const syncRules = await this.factory.getNextSyncRulesContent(); + const syncRules = await this.factory.getDeployingSyncConfigContent(); if (syncRules == null) { throw new Error(`Next replication stream not available`); } this.syncRulesContent = syncRules; - this.storage = this.factory.getInstance(syncRules); + const replicationStream = await this.factory.getReplicationStream(syncRules.replicationStreamId); + if (replicationStream == null) { + throw new Error(`Next replication stream not available`); + } + this.storage = this.factory.getInstance(replicationStream); return this.storage!; } async loadActiveSyncRules() { - const syncRules = await this.factory.getActiveSyncRulesContent(); + const syncRules = await this.factory.getActiveSyncConfigContent(); if (syncRules == null) { throw new Error(`Active replication stream not available`); } this.syncRulesContent = syncRules; - this.storage = this.factory.getInstance(syncRules); + const replicationStream = await this.factory.getReplicationStream(syncRules.replicationStreamId); + if (replicationStream == null) { + throw new Error(`Active replication stream not available`); + } + this.storage = this.factory.getInstance(replicationStream); return this.storage!; } - private getSyncRulesContent(): storage.PersistedSyncRulesContent { + private getSyncRulesContent(): storage.PersistedSyncConfigContent { if (this.syncRulesContent == null) { throw new Error('Sync config not configured - call updateSyncRules() first'); } diff --git a/packages/service-core-tests/src/test-utils/AbstractStreamTestContext.ts b/packages/service-core-tests/src/test-utils/AbstractStreamTestContext.ts index 33265f124..e1ee2ea40 100644 --- a/packages/service-core-tests/src/test-utils/AbstractStreamTestContext.ts +++ b/packages/service-core-tests/src/test-utils/AbstractStreamTestContext.ts @@ -14,7 +14,8 @@ import { fromAsync } from './stream_utils.js'; export abstract class AbstractStreamTestContext implements AsyncDisposable { protected abortController = new AbortController(); - protected syncRulesContent?: storage.PersistedSyncRulesContent; + protected syncRulesContent?: storage.PersistedSyncConfigContent; + protected replicationStream?: storage.PersistedReplicationStream; public storage?: SyncRulesBucketStorage; protected settledReplicationPromise?: Promise>; @@ -44,37 +45,47 @@ export abstract class AbstractStreamTestContext implements AsyncDisposable { } async updateSyncRules(content: string) { - const syncRules = await this.factory.updateSyncRules( + const stream = await this.factory.updateSyncRules( updateSyncRulesFromYaml(content, { validate: true, storageVersion: this.storageVersion }) ); - this.syncRulesContent = syncRules; - this.storage = this.factory.getInstance(syncRules); + this.replicationStream = stream; + this.syncRulesContent = (await this.factory.getReplicationStreamConfigs(stream.id))[0]; + this.storage = this.factory.getInstance(stream); return this.storage!; } async loadNextSyncRules() { - const syncRules = await this.factory.getNextSyncRulesContent(); + const syncRules = await this.factory.getDeployingSyncConfigContent(); if (syncRules == null) { throw new Error(`Next sync rules not available`); } + const stream = await this.factory.getReplicationStream(syncRules.replicationStreamId); + if (stream == null) { + throw new Error(`Next replication stream not available`); + } this.syncRulesContent = syncRules; - this.storage = this.factory.getInstance(syncRules); + this.replicationStream = stream; + this.storage = this.factory.getInstance(stream); return this.storage!; } async loadActiveSyncRules() { - const syncRules = await this.factory.getActiveSyncRulesContent(); + const syncRules = await this.factory.getActiveSyncConfigContent(); if (syncRules == null) { throw new Error(`Active sync rules not available`); } + const storage = await this.factory.getActiveStorage(); + if (storage == null) { + throw new Error(`Active replication stream not available`); + } this.syncRulesContent = syncRules; - this.storage = this.factory.getInstance(syncRules); + this.storage = storage; return this.storage!; } - private getSyncRulesContent(): storage.PersistedSyncRulesContent { + private getSyncRulesContent(): storage.PersistedSyncConfigContent { if (this.syncRulesContent == null) { throw new Error('Sync rules not configured - call updateSyncRules() first'); } diff --git a/packages/service-core-tests/src/test-utils/StorageDataHelpers.ts b/packages/service-core-tests/src/test-utils/StorageDataHelpers.ts index 20363e5ea..28f88e298 100644 --- a/packages/service-core-tests/src/test-utils/StorageDataHelpers.ts +++ b/packages/service-core-tests/src/test-utils/StorageDataHelpers.ts @@ -1,8 +1,8 @@ import { InternalOpId, OplogEntry, + PersistedSyncConfigContent, PersistedSyncRules, - PersistedSyncRulesContent, SyncRulesBucketStorage } from '@powersync/service-core'; import { bucketRequest } from './general-utils.js'; @@ -10,9 +10,9 @@ import { fromAsync } from './stream_utils.js'; export class StorageDataHelpers { storage: SyncRulesBucketStorage; - syncRules: PersistedSyncRulesContent | PersistedSyncRules; + syncRules: PersistedSyncConfigContent | PersistedSyncRules; - constructor(storage: SyncRulesBucketStorage, syncRules: PersistedSyncRulesContent | PersistedSyncRules) { + constructor(storage: SyncRulesBucketStorage, syncRules: PersistedSyncConfigContent | PersistedSyncRules) { this.storage = storage; this.syncRules = syncRules; } diff --git a/packages/service-core-tests/src/test-utils/general-utils.ts b/packages/service-core-tests/src/test-utils/general-utils.ts index 89df81840..325a22b51 100644 --- a/packages/service-core-tests/src/test-utils/general-utils.ts +++ b/packages/service-core-tests/src/test-utils/general-utils.ts @@ -14,6 +14,21 @@ export const BATCH_OPTIONS: storage.CreateWriterOptions = { storeCurrentData: true }; +/** + * Deploy a sync config and return both the replication stream (for {@link storage.BucketStorageFactory.getInstance}) + * and the deployed config content (for parsing / bucket requests). + * + * Replication streams and sync config content are separate concerns since one stream can hold multiple configs. + */ +export async function deploySyncRules( + factory: storage.BucketStorageFactory, + options: storage.UpdateSyncRulesOptions +): Promise<{ stream: storage.PersistedReplicationStream; content: storage.PersistedSyncConfigContent }> { + const stream = await factory.updateSyncRules(options); + const content = (await factory.getReplicationStreamConfigs(stream.id))[0]; + return { stream, content }; +} + /** * With newer storage versions, we need actual test tables, resolved via the writer. */ @@ -78,9 +93,9 @@ export function getBatchData( } function isParsedSyncRules( - syncRules: storage.PersistedSyncRulesContent | storage.PersistedSyncRules + syncRules: storage.PersistedSyncConfigContent | storage.PersistedSyncRules ): syncRules is storage.PersistedSyncRules { - return (syncRules as storage.PersistedSyncRules).syncConfigWithErrors !== undefined; + return (syncRules as storage.PersistedSyncRules).syncConfigs !== undefined; } /** @@ -88,7 +103,7 @@ function isParsedSyncRules( * This converts a bucket name like "global[]" into the actual bucket name, for use in tests. */ export function bucketRequest( - syncRules: storage.PersistedSyncRulesContent | storage.PersistedSyncRules, + syncRules: storage.PersistedSyncConfigContent | storage.PersistedSyncRules, bucket: string, start?: InternalOpId | string | number ): BucketDataRequest { @@ -97,7 +112,9 @@ export function bucketRequest( const parameterStart = bucket.indexOf('['); const definitionName = bucket.substring(0, parameterStart); const parameters = bucket.substring(parameterStart); - const source = parsed.syncConfigWithErrors.config.bucketDataSources.find((b) => b.uniqueName === definitionName); + const source = parsed.syncConfigs + .flatMap((config) => config.config.bucketDataSources) + .find((b) => b.uniqueName === definitionName); if (source == null) { throw new Error(`Failed to find global bucket ${bucket}`); diff --git a/packages/service-core-tests/src/tests/register-compacting-tests.ts b/packages/service-core-tests/src/tests/register-compacting-tests.ts index 97b71ba79..db027db95 100644 --- a/packages/service-core-tests/src/tests/register-compacting-tests.ts +++ b/packages/service-core-tests/src/tests/register-compacting-tests.ts @@ -17,6 +17,7 @@ bucket_definitions: `) ); const bucketStorage = factory.getInstance(syncRules); + const syncRulesContent = (await factory.getReplicationStreamConfigs(syncRules.id))[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -53,7 +54,7 @@ bucket_definitions: const checkpoint = writer.last_flushed_op!; - const request = bucketRequest(syncRules, 'global[]'); + const request = bucketRequest(syncRulesContent, 'global[]'); const batchBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request])); const dataBefore = batchBefore.chunkData.data; @@ -124,6 +125,7 @@ bucket_definitions: `) ); const bucketStorage = factory.getInstance(syncRules); + const syncRulesContent = (await factory.getReplicationStreamConfigs(syncRules.id))[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -168,7 +170,7 @@ bucket_definitions: await writer.flush(); const checkpoint = writer.last_flushed_op!; - const request = bucketRequest(syncRules, 'global[]'); + const request = bucketRequest(syncRulesContent, 'global[]'); const batchBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request])); const dataBefore = batchBefore.chunkData.data; @@ -240,6 +242,7 @@ bucket_definitions: `) ); const bucketStorage = factory.getInstance(syncRules); + const syncRulesContent = (await factory.getReplicationStreamConfigs(syncRules.id))[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -275,7 +278,7 @@ bucket_definitions: await writer.flush(); const checkpoint1 = writer.last_flushed_op!; - const request = bucketRequest(syncRules, 'global[]'); + const request = bucketRequest(syncRulesContent, 'global[]'); await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config); await writer2.save({ @@ -328,6 +331,7 @@ bucket_definitions: - select * from test where b = bucket.b`) ); const bucketStorage = factory.getInstance(syncRules); + const syncRulesContent = (await factory.getReplicationStreamConfigs(syncRules.id))[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -413,7 +417,7 @@ bucket_definitions: const batchAfter = await test_utils.fromAsync( bucketStorage.getBucketDataBatch( checkpoint, - bucketRequestMap(syncRules, [ + bucketRequestMap(syncRulesContent, [ ['grouped["b1"]', 0n], ['grouped["b2"]', 0n] ]) @@ -458,6 +462,7 @@ bucket_definitions: `) ); const bucketStorage = factory.getInstance(syncRules); + const syncRulesContent = (await factory.getReplicationStreamConfigs(syncRules.id))[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -513,7 +518,7 @@ bucket_definitions: await writer2.commit('2/1'); await writer2.flush(); const checkpoint2 = writer2.last_flushed_op!; - const request = bucketRequest(syncRules, 'global[]'); + const request = bucketRequest(syncRulesContent, 'global[]'); await bucketStorage.clearChecksumCache(); const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]); const globalChecksum = checksumAfter.get(request.bucket); @@ -536,6 +541,7 @@ bucket_definitions: `) ); const bucketStorage = factory.getInstance(syncRules); + const syncRulesContent = (await factory.getReplicationStreamConfigs(syncRules.id))[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -562,7 +568,7 @@ bucket_definitions: await writer.flush(); // Get checksums here just to populate the cache - await bucketStorage.getChecksums(writer.last_flushed_op!, bucketRequests(syncRules, ['global[]'])); + await bucketStorage.getChecksums(writer.last_flushed_op!, bucketRequests(syncRulesContent, ['global[]'])); await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable2 = await test_utils.resolveTestTable(writer2, 'test', ['id'], config); await writer2.save({ @@ -585,7 +591,7 @@ bucket_definitions: }); const checkpoint2 = writer2.last_flushed_op!; - const request = bucketRequest(syncRules, 'global[]'); + const request = bucketRequest(syncRulesContent, 'global[]'); // Check that the checksum was correctly updated with the clear operation after having a cached checksum const checksumAfter = await bucketStorage.getChecksums(checkpoint2, [request]); const globalChecksum = checksumAfter.get(request.bucket); @@ -607,6 +613,7 @@ bucket_definitions: `) ); const bucketStorage = factory.getInstance(syncRules); + const syncRulesContent = (await factory.getReplicationStreamConfigs(syncRules.id))[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -646,7 +653,7 @@ bucket_definitions: }); const batchAfterDefaultCompact = await test_utils.oneFromAsync( - bucketStorage.getBucketDataBatch(checkpoint2, bucketRequestMap(syncRules, [['global[]', 0n]])) + bucketStorage.getBucketDataBatch(checkpoint2, bucketRequestMap(syncRulesContent, [['global[]', 0n]])) ); // Operation 1 should remain a PUT because op_id=2 is above the default maxOpId checkpoint. diff --git a/packages/service-core-tests/src/tests/register-data-storage-data-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-data-tests.ts index 1e124835d..73fa22cbb 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-data-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-data-tests.ts @@ -38,7 +38,8 @@ export function registerDataStorageDataTests(config: storage.TestStorageConfig) test('removing row', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -49,7 +50,7 @@ bucket_definitions: { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -103,7 +104,8 @@ bucket_definitions: test('insert after delete in new batch', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -114,7 +116,7 @@ bucket_definitions: { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); @@ -167,7 +169,8 @@ bucket_definitions: test('update after delete in new batch', async () => { // Update after delete may not be common, but the storage layer should handle it in an eventually-consistent way. await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -178,7 +181,7 @@ bucket_definitions: { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); @@ -234,7 +237,8 @@ bucket_definitions: test('insert after delete in same batch', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -247,7 +251,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); @@ -296,7 +300,8 @@ bucket_definitions: test('(insert, delete, insert), (delete)', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -309,7 +314,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); { await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -374,7 +379,8 @@ bucket_definitions: test('changing client ids', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -387,7 +393,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); @@ -445,7 +451,8 @@ bucket_definitions: test('re-apply delete', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -458,7 +465,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); @@ -521,7 +528,8 @@ bucket_definitions: test('re-apply update + delete', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -532,7 +540,7 @@ bucket_definitions: { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); @@ -651,7 +659,8 @@ bucket_definitions: // 2. Output order not being correct. await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -662,7 +671,7 @@ bucket_definitions: { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -810,7 +819,8 @@ bucket_definitions: ]); } await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -823,7 +833,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id', 'description'], config); @@ -923,7 +933,8 @@ bucket_definitions: } await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -936,7 +947,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id', 'description'], config); @@ -1026,7 +1037,8 @@ bucket_definitions: // The specific batch splits is an implementation detail of the storage driver, // and the test will have to updated when other implementations are added. await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -1039,7 +1051,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); @@ -1138,7 +1150,8 @@ bucket_definitions: test('long batch', async () => { // Test syncing a batch of data that is limited by count. await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -1151,7 +1164,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); @@ -1217,7 +1230,8 @@ bucket_definitions: describe('batch has_more', () => { const setup = async (options: BucketDataBatchOptions) => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -1231,7 +1245,7 @@ bucket_definitions: { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); @@ -1392,7 +1406,8 @@ bucket_definitions: // but large enough in size to be split over multiple returned chunks. // Similar to the above test, but splits over 1MB chunks. await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -1406,7 +1421,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config, 1); @@ -1444,7 +1459,8 @@ bucket_definitions: test('unchanged checksums', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -1457,7 +1473,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -1485,7 +1501,8 @@ bucket_definitions: test('empty checkpoints (1)', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -1496,7 +1513,7 @@ bucket_definitions: { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('1/1'); await writer.commit('1/1'); @@ -1521,7 +1538,8 @@ bucket_definitions: test('empty checkpoints (2)', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -1534,7 +1552,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); await using writer1 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer2, 'test', ['id'], config); @@ -1571,7 +1589,8 @@ bucket_definitions: test('empty checkpoints (sync rule activation)', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -1584,7 +1603,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const result1 = await writer.commit('1/1', { createEmptyCheckpoints: false }); @@ -1609,7 +1628,7 @@ bucket_definitions: const cp2 = await bucketStorage.getCheckpoint(); expect(cp2.lsn).toEqual('3/1'); - const activeSyncRules = await factory.getActiveSyncRulesContent(); + const activeSyncRules = await factory.getActiveSyncConfigContent(); expect(activeSyncRules?.id).toEqual(syncRules.id); // At this point, it should be a truely empty checkpoint @@ -1623,7 +1642,8 @@ bucket_definitions: test.runIf(storageVersion >= 3)('deleting while streaming', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -1636,7 +1656,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); await using snapshotWriter = await bucketStorage.createWriter({ ...test_utils.BATCH_OPTIONS, skipExistingRows: true @@ -1692,7 +1712,8 @@ export function testChecksumBatching(config: storage.TestStorageConfig) { const storageVersion = config.storageVersion ?? CURRENT_STORAGE_VERSION; test('checksums for multiple buckets', async () => { await using factory = await config.factory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -1706,7 +1727,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); @@ -1756,7 +1777,11 @@ streams: `) ); - const { errors } = deployed.parsed({ defaultSchema: 'ignored' }).syncConfigWithErrors; + const [deployedContent] = await factory.getReplicationStreamConfigs(deployed.id); + expect(deployedContent).toBeDefined(); + const [deployedConfig] = deployedContent.parsed({ defaultSchema: 'ignored' }).syncConfigs; + expect(deployedConfig).toBeDefined(); + const { errors } = deployedConfig; expect(errors).toHaveLength(1); expect(errors[0].message).toStrictEqual('Expected a SELECT statement'); expect(errors[0].location).toStrictEqual({ diff --git a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts index 8610c2133..56009657d 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts @@ -26,7 +26,8 @@ export function registerDataStorageParameterTests(config: storage.TestStorageCon test('save and load parameters', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -40,7 +41,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -92,7 +93,8 @@ bucket_definitions: test('it should use the latest version', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -106,7 +108,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -163,7 +165,8 @@ bucket_definitions: test('it should use the latest version after updates', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -177,7 +180,7 @@ bucket_definitions: { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -250,7 +253,8 @@ bucket_definitions: test('save and load parameters with different number types', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -264,7 +268,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -318,7 +322,8 @@ bucket_definitions: // test this to ensure correct deserialization. await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -332,7 +337,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -385,7 +390,8 @@ bucket_definitions: test('save and load parameters with workspaceId', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -401,7 +407,7 @@ bucket_definitions: ) ); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig(); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const workspaceTable = await test_utils.resolveTestTable(writer, 'workspace', ['id'], config); @@ -443,7 +449,8 @@ bucket_definitions: test('save and load parameters with dynamic global buckets', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -459,7 +466,7 @@ bucket_definitions: ) ); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig(); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const workspaceTable = await test_utils.resolveTestTable(writer, 'workspace', undefined, config); @@ -535,7 +542,8 @@ bucket_definitions: test('multiple parameter queries', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -553,7 +561,7 @@ bucket_definitions: ) ); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig(); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const workspaceTable = await test_utils.resolveTestTable(writer, 'workspace', undefined, config); @@ -639,7 +647,8 @@ bucket_definitions: test('truncate parameters', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` bucket_definitions: @@ -653,7 +662,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -732,7 +741,8 @@ bucket_definitions: test('sync streams smoke test', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml(` config: edition: 3 @@ -744,7 +754,7 @@ streams: WHERE data.foo = param.bar AND param.baz = auth.user_id() `) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -799,7 +809,8 @@ streams: test('respects parameter limit', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` config: @@ -815,7 +826,7 @@ streams: } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -851,7 +862,8 @@ streams: test('sync streams store multiple parameter outputs for a single source row and lookup', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml(` config: edition: 3 @@ -864,7 +876,7 @@ streams: WHERE a.x = x.value AND y.value = auth.user_id() `) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -979,7 +991,8 @@ streams: test('can request multiple lookups at once', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml(` config: edition: 3 @@ -992,10 +1005,12 @@ streams: query: SELECT * FROM b WHERE p IN (SELECT id FROM param_b WHERE u = auth.user_id()) `) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); const parsedSyncRules = syncRules.parsed(test_utils.PARSE_OPTIONS); const hydrationState = parsedSyncRules.hydrationState; - const syncConfig = parsedSyncRules.syncConfigWithErrors.config; + const [parsedSyncConfig] = parsedSyncRules.syncConfigs; + expect(parsedSyncConfig).toBeDefined(); + const syncConfig = parsedSyncConfig.config; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const paramATable = await test_utils.resolveTestTable(writer, 'param_a', ['id'], config, 1); @@ -1069,7 +1084,8 @@ streams: test('sync streams preserve duplicate downstream lookups with different provenance', async () => { await using factory = await generateStorageFactory(); - const syncRules = await factory.updateSyncRules( + const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( + factory, updateSyncRulesFromYaml( ` config: @@ -1090,7 +1106,7 @@ streams: } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = factory.getInstance(replicationStream); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); diff --git a/packages/service-core-tests/src/tests/register-sync-tests.ts b/packages/service-core-tests/src/tests/register-sync-tests.ts index fdbf4b1dc..cdcfbaf89 100644 --- a/packages/service-core-tests/src/tests/register-sync-tests.ts +++ b/packages/service-core-tests/src/tests/register-sync-tests.ts @@ -892,7 +892,8 @@ bucket_definitions: const checkpoint2 = await getCheckpointLines(iter); - const { bucket } = test_utils.bucketRequest(syncRules, 'by_user["user1"]'); + const syncRulesContent = (await f.getReplicationStreamConfigs(syncRules.id))[0]; + const { bucket } = test_utils.bucketRequest(syncRulesContent, 'by_user["user1"]'); expect( (checkpoint2[0] as StreamingSyncCheckpointDiff).checkpoint_diff?.updated_buckets?.map((b) => b.bucket) ).toEqual([bucket]); @@ -947,7 +948,8 @@ bucket_definitions: iter.return?.(); }); - const { bucket } = bucketRequest(syncRules, 'by_user["user1"]'); + const syncRulesContent = (await f.getReplicationStreamConfigs(syncRules.id))[0]; + const { bucket } = bucketRequest(syncRulesContent, 'by_user["user1"]'); const checkpoint1 = await getCheckpointLines(iter); expect((checkpoint1[0] as StreamingSyncCheckpoint).checkpoint?.buckets?.map((b) => b.bucket)).toEqual([bucket]); @@ -1038,7 +1040,8 @@ bucket_definitions: await writer.commit('0/1'); - const { bucket } = test_utils.bucketRequest(syncRules, 'by_user["user1"]'); + const syncRulesContent = (await f.getReplicationStreamConfigs(syncRules.id))[0]; + const { bucket } = test_utils.bucketRequest(syncRulesContent, 'by_user["user1"]'); const checkpoint2 = await getCheckpointLines(iter); expect( diff --git a/packages/service-core-tests/src/tests/util.ts b/packages/service-core-tests/src/tests/util.ts index c056ee6a7..9e5c99e83 100644 --- a/packages/service-core-tests/src/tests/util.ts +++ b/packages/service-core-tests/src/tests/util.ts @@ -9,14 +9,14 @@ import { import { bucketRequest } from '../test-utils/general-utils.js'; export function bucketRequestMap( - syncRules: storage.PersistedSyncRulesContent, + syncRules: storage.PersistedSyncConfigContent, buckets: Iterable ): storage.BucketDataRequest[] { return Array.from(buckets, ([bucketName, opId]) => bucketRequest(syncRules, bucketName, opId)); } export function bucketRequests( - syncRules: storage.PersistedSyncRulesContent, + syncRules: storage.PersistedSyncConfigContent, bucketNames: string[] ): storage.BucketChecksumRequest[] { return bucketNames.map((bucketName) => { diff --git a/packages/service-core/src/api/diagnostics.ts b/packages/service-core/src/api/diagnostics.ts index 20098bdca..f0f0a9fb5 100644 --- a/packages/service-core/src/api/diagnostics.ts +++ b/packages/service-core/src/api/diagnostics.ts @@ -1,4 +1,4 @@ -import { logger } from '@powersync/lib-services-framework'; +import { logger, ServiceAssertionError } from '@powersync/lib-services-framework'; import { DEFAULT_TAG, SourceTableRef, SyncConfigWithErrors } from '@powersync/service-sync-rules'; import { ReplicationError, SyncRulesStatus, TableInfo } from '@powersync/service-types'; @@ -30,8 +30,16 @@ export const DEFAULT_DATASOURCE_ID = 'default'; export async function getSyncRulesStatus( bucketStorage: storage.BucketStorageFactory, apiHandler: RouteAPI, - sync_rules: storage.PersistedSyncRulesContent | null, - options: DiagnosticsOptions + sync_rules: storage.PersistedSyncConfigContent | null, + options: DiagnosticsOptions, + syncConfigStatus?: storage.PersistedSyncConfigStatus | null, + /** + * Storage instance for the replication stream of this config. + * + * Required to populate live status (snapshot/checkpoint info). The content object + * itself is no longer a replication stream, so the caller must resolve this. + */ + systemStorage?: storage.SyncRulesBucketStorage ): Promise { if (sync_rules == null) { return undefined; @@ -46,7 +54,12 @@ export async function getSyncRulesStatus( let persisted: storage.PersistedSyncRules; try { persisted = sync_rules.parsed(apiHandler.getParseSyncRulesOptions()); - parsed = persisted.syncConfigWithErrors; + // A content object represents a single sync config, so its parsed result has exactly one entry. + const [singleConfig] = persisted.syncConfigs; + if (singleConfig == null) { + throw new ServiceAssertionError('Expected one sync config'); + } + parsed = singleConfig; } catch (e) { return { content: include_content ? sync_rules.sync_rules_content : undefined, @@ -60,8 +73,7 @@ export async function getSyncRulesStatus( // This method can run under some situations if no connection is configured yet. // It will return a default tag in such a case. This default tag is not module specific. const tag = sourceConfig.tag ?? DEFAULT_TAG; - const systemStorage = live_status ? bucketStorage.getInstance(sync_rules) : undefined; - const status = await systemStorage?.getStatus(); + const status = live_status ? await systemStorage?.getStatus() : undefined; let replication_lag_bytes: number | undefined = undefined; let slot_wal_budget: SlotWalBudgetInfo | undefined = undefined; @@ -136,11 +148,13 @@ export async function getSyncRulesStatus( } const errors = tables_flat.flatMap((info) => info.errors); - if (sync_rules.last_fatal_error) { + const statusSource = syncConfigStatus ?? sync_rules; + + if (statusSource.last_fatal_error) { errors.push({ level: 'fatal', - message: sync_rules.last_fatal_error, - ts: sync_rules.last_fatal_error_ts?.toISOString() + message: statusSource.last_fatal_error, + ts: statusSource.last_fatal_error_ts?.toISOString() }); } errors.push(...syncRuleErrors.map((error) => syncConfigYamlErrorToReplicationError(error, now))); @@ -171,7 +185,7 @@ export async function getSyncRulesStatus( if (live_status && status?.active) { // Check replication lag for active replication stream. // Right now we exclude mysql, since it we don't have consistent keepalives for it. - if (sync_rules.last_checkpoint_ts == null && sync_rules.last_keepalive_ts == null) { + if (statusSource.last_checkpoint_ts == null && statusSource.last_keepalive_ts == null) { errors.push({ level: 'warning', message: 'No checkpoint found, cannot calculate replication lag', @@ -179,8 +193,8 @@ export async function getSyncRulesStatus( }); } else { const lastTime = Math.max( - sync_rules.last_checkpoint_ts?.getTime() ?? 0, - sync_rules.last_keepalive_ts?.getTime() ?? 0 + statusSource.last_checkpoint_ts?.getTime() ?? 0, + statusSource.last_keepalive_ts?.getTime() ?? 0 ); const lagSeconds = Math.round((Date.now() - lastTime) / 1000); // On idle instances, keepalive messages are only persisted every 60 seconds. @@ -213,8 +227,8 @@ export async function getSyncRulesStatus( initial_replication_done: status?.snapshot_done ?? false, // TODO: Rename? last_lsn: status?.checkpoint_lsn ?? undefined, - last_checkpoint_ts: sync_rules.last_checkpoint_ts?.toISOString(), - last_keepalive_ts: sync_rules.last_keepalive_ts?.toISOString(), + last_checkpoint_ts: statusSource.last_checkpoint_ts?.toISOString(), + last_keepalive_ts: statusSource.last_keepalive_ts?.toISOString(), replication_lag_bytes: replication_lag_bytes, wal_status: slot_wal_budget?.wal_status, safe_wal_size: slot_wal_budget?.safe_wal_size, diff --git a/packages/service-core/src/modules/AbstractModule.ts b/packages/service-core/src/modules/AbstractModule.ts index 848966284..002d0baf0 100644 --- a/packages/service-core/src/modules/AbstractModule.ts +++ b/packages/service-core/src/modules/AbstractModule.ts @@ -1,13 +1,17 @@ import { logger } from '@powersync/lib-services-framework'; import winston from 'winston'; -import { PersistedSyncRulesContent } from '../storage/storage-index.js'; +import { PersistedReplicationStream } from '../storage/storage-index.js'; import { ServiceContextContainer } from '../system/ServiceContext.js'; export interface TearDownOptions { /** * If required, tear down any configuration/state for the specific replication stream */ - syncRules?: PersistedSyncRulesContent[]; + replicationStreams?: PersistedReplicationStream[]; + /** + * @deprecated Use replicationStreams. + */ + syncRules?: PersistedReplicationStream[]; } export interface AbstractModuleOptions { diff --git a/packages/service-core/src/replication/AbstractReplicator.ts b/packages/service-core/src/replication/AbstractReplicator.ts index 924c358de..bebdec10d 100644 --- a/packages/service-core/src/replication/AbstractReplicator.ts +++ b/packages/service-core/src/replication/AbstractReplicator.ts @@ -38,10 +38,10 @@ export abstract class AbstractReplicator(); + private replicationJobs = new Map(); /** * Map of replciation stream ids to promises that are clearing the replication stream. @@ -188,48 +188,49 @@ export abstract class AbstractReplicator(this.replicationJobs.entries()); - const replicatingSyncRules = await this.storage.getReplicatingSyncRules(); - const newJobs = new Map(); + const existingJobs = new Map(this.replicationJobs.entries()); + const replicatingStreams = await this.storage.getReplicatingReplicationStreams(); + const newJobs = new Map(); let activeJob: T | undefined = undefined; - for (let syncRules of replicatingSyncRules) { - const existingJob = existingJobs.get(syncRules.id); - if (syncRules.active && activeJob == null) { + for (let replicationStream of replicatingStreams) { + const jobId = replicationStream.replicationJobId; + const existingJob = existingJobs.get(jobId); + if (replicationStream.state == storage.SyncRuleState.ACTIVE && activeJob == null) { activeJob = existingJob; } if (existingJob && !existingJob.isStopped) { // No change - existingJobs.delete(syncRules.id); - newJobs.set(syncRules.id, existingJob); + existingJobs.delete(jobId); + newJobs.set(jobId, existingJob); } else if (existingJob && existingJob.isStopped) { // Stopped (e.g. fatal error). // Remove from the list. Next refresh call will restart the job. - existingJobs.delete(syncRules.id); + existingJobs.delete(jobId); } else { // New sync config was found (or resume after restart) try { let lock: storage.ReplicationLock; - if (configuredLock?.sync_rules_id == syncRules.id) { + if (configuredLock?.sync_rules_id == replicationStream.id) { lock = configuredLock; } else { - lock = await syncRules.lock(); + lock = await replicationStream.lock(); } - const storage = this.storage.getInstance(syncRules); + const syncRuleStorage = this.storage.getInstance(replicationStream); const newJob = this.createJob({ lock: lock, - storage: storage + storage: syncRuleStorage }); - newJobs.set(syncRules.id, newJob); + newJobs.set(jobId, newJob); newJob.start(); - if (syncRules.active) { + if (replicationStream.state == storage.SyncRuleState.ACTIVE) { activeJob = newJob; } this.lockAlerted = false; } catch (e) { if (e?.errorData?.code === ErrorCode.PSYNC_S1003) { if (!this.lockAlerted) { - syncRules.logger.info(`[${e.errorData.code}] ${e.errorData.description}`); + replicationStream.logger.info(`[${e.errorData.code}] ${e.errorData.description}`); this.lockAlerted = true; } } else { @@ -237,7 +238,7 @@ export abstract class AbstractReplicator { syncRuleStorage.logger.warn(`Failed clean up replication config`, e); }) .finally(() => { - this.clearingJobs.delete(syncRules.id); + this.clearingJobs.delete(replicationStream.id); }); - this.clearingJobs.set(syncRules.id, promise); + this.clearingJobs.set(replicationStream.id, promise); } } diff --git a/packages/service-core/src/routes/endpoints/admin.ts b/packages/service-core/src/routes/endpoints/admin.ts index bd890146e..737864205 100644 --- a/packages/service-core/src/routes/endpoints/admin.ts +++ b/packages/service-core/src/routes/endpoints/admin.ts @@ -65,20 +65,43 @@ export const diagnostics = routeDefinition({ const { storageEngine: { activeBucketStorage } } = service_context; - const active = await activeBucketStorage.getActiveSyncRulesContent(); - const next = await activeBucketStorage.getNextSyncRulesContent(); + const active = await activeBucketStorage.getActiveSyncConfigContent(); + const activeConfigStatus = await activeBucketStorage.getActiveSyncConfigStatus(); + const activeStorage = await activeBucketStorage.getActiveStorage(); + const deploying = await activeBucketStorage.getDeployingSyncConfigContent(); - const active_status = await api.getSyncRulesStatus(activeBucketStorage, apiHandler, active, { - include_content, - check_connection: status.connected, - live_status: true - }); + const active_status = await api.getSyncRulesStatus( + activeBucketStorage, + apiHandler, + active, + { + include_content, + check_connection: status.connected, + live_status: true + }, + activeConfigStatus, + activeStorage ?? undefined + ); - const next_status = await api.getSyncRulesStatus(activeBucketStorage, apiHandler, next, { - include_content, - check_connection: status.connected, - live_status: true - }); + const deploying_status = + deploying == null + ? undefined + : await (async (syncConfig) => { + const stream = await activeBucketStorage.getReplicationStream(syncConfig.replicationStreamId); + const systemStorage = stream == null ? undefined : activeBucketStorage.getInstance(stream); + return api.getSyncRulesStatus( + activeBucketStorage, + apiHandler, + syncConfig, + { + include_content, + check_connection: status.connected, + live_status: true + }, + syncConfig.getSyncConfigStatus(), + systemStorage + ); + })(deploying); return internal_routes.DiagnosticsResponse.encode({ connections: [ @@ -89,7 +112,7 @@ export const diagnostics = routeDefinition({ } ], active_sync_rules: active_status, - deploying_sync_rules: next_status + deploying_sync_rules: deploying_status }); } }); @@ -119,12 +142,16 @@ export const reprocess = routeDefinition({ storageEngine: { activeBucketStorage } } = service_context; const apiHandler = service_context.routerEngine.getAPI(); - const next = await activeBucketStorage.getNextSyncRules(apiHandler.getParseSyncRulesOptions()); + const next = await activeBucketStorage.getDeployingSyncConfigContent(); if (next != null) { - throw new Error(`Busy processing sync config - cannot reprocess`); + throw new errors.ServiceError({ + status: 409, + code: ErrorCode.PSYNC_S4106, + description: 'Busy processing sync config - cannot reprocess' + }); } - const active = await activeBucketStorage.getActiveSyncRules(apiHandler.getParseSyncRulesOptions()); + const active = await activeBucketStorage.getActiveSyncConfigContent(); if (active == null) { throw new errors.ServiceError({ status: 422, @@ -132,13 +159,12 @@ export const reprocess = routeDefinition({ description: 'No active sync config' }); } - // There are some differences between this and using asUpdateOptions(): // 1. This always re-parses the source YAML. If there are changes to the sync stream compiler, that can affect the sync plan. // 2. If the source does not set the storage version, this will update it do the current version. // We can consider tweaking this behavior in the future. const new_rules = await activeBucketStorage.updateSyncRules( - storage.updateSyncRulesFromYaml(active.syncConfigWithErrors.config.content, { + storage.updateSyncRulesFromYaml(active.sync_rules_content, { // This sync config already passed validation. But if the config is not valid anymore due // to a service change, we do want to report the error here. validate: true @@ -160,31 +186,28 @@ export const reprocess = routeDefinition({ } }); -class FakeSyncRulesContentForValidation extends storage.PersistedSyncRulesContent { +class FakeSyncRulesContentForValidation extends storage.PersistedSyncConfigContent { constructor( private readonly apiHandler: api.RouteAPI, private readonly schema: SourceSchema, - data: storage.PersistedSyncRulesContentData + data: storage.PersistedSyncConfigContentData ) { super(data); } - current_lock: storage.ReplicationLock | null = null; - - async lock(): Promise { - throw new Error('Lock not implemented'); - } - parsed(options: storage.ParseSyncRulesOptions): storage.PersistedSyncRules { + const syncConfig = SqlSyncRules.fromYaml(this.sync_rules_content, { + ...this.apiHandler.getParseSyncRulesOptions(), + schema: this.schema + }); + return { - ...this, - syncConfigWithErrors: SqlSyncRules.fromYaml(this.sync_rules_content, { - ...this.apiHandler.getParseSyncRulesOptions(), - schema: this.schema - }), + id: this.id, + slot_name: this.slot_name, + syncConfigs: [syncConfig], hydrationState: DEFAULT_HYDRATION_STATE, hydratedSyncConfig() { - return this.syncConfigWithErrors.config.hydrate({ + return syncConfig.config.hydrate({ hydrationState: DEFAULT_HYDRATION_STATE, sqlite: nodeSqlite(sqlite) }); diff --git a/packages/service-core/src/routes/endpoints/sync-rules.ts b/packages/service-core/src/routes/endpoints/sync-rules.ts index 5db4da611..3f11deb20 100644 --- a/packages/service-core/src/routes/endpoints/sync-rules.ts +++ b/packages/service-core/src/routes/endpoints/sync-rules.ts @@ -110,7 +110,7 @@ export const currentSyncRules = routeDefinition({ storageEngine: { activeBucketStorage } } = service_context; - const sync_rules = await activeBucketStorage.getActiveSyncRulesContent(); + const sync_rules = await activeBucketStorage.getActiveSyncConfigContent(); if (!sync_rules) { throw new errors.ServiceError({ status: 422, @@ -121,7 +121,7 @@ export const currentSyncRules = routeDefinition({ const apiHandler = service_context.routerEngine.getAPI(); const info = await debugSyncRules(apiHandler, sync_rules.sync_rules_content); - const next = await activeBucketStorage.getNextSyncRulesContent(); + const next = await activeBucketStorage.getDeployingSyncConfigContent(); const next_info = next ? await debugSyncRules(apiHandler, next.sync_rules_content) : null; @@ -156,8 +156,16 @@ export const reprocessSyncRules = routeDefinition({ const { storageEngine: { activeBucketStorage } } = payload.context.service_context; - const apiHandler = payload.context.service_context.routerEngine.getAPI(); - const sync_rules = await activeBucketStorage.getActiveSyncRules(apiHandler.getParseSyncRulesOptions()); + const next = await activeBucketStorage.getDeployingSyncConfigContent(); + if (next != null) { + throw new errors.ServiceError({ + status: 409, + code: ErrorCode.PSYNC_S4106, + description: 'Busy processing sync config - cannot reprocess' + }); + } + + const sync_rules = await activeBucketStorage.getActiveSyncConfigContent(); if (sync_rules == null) { throw new errors.ServiceError({ status: 422, @@ -167,7 +175,7 @@ export const reprocessSyncRules = routeDefinition({ } const new_rules = await activeBucketStorage.updateSyncRules( - updateSyncRulesFromYaml(sync_rules.syncConfigWithErrors.config.content, { + updateSyncRulesFromYaml(sync_rules.sync_rules_content, { // This sync config already passed validation. But if the rules are not valid anymore due // to a service change, we do want to report the error here. validate: true diff --git a/packages/service-core/src/runner/teardown.ts b/packages/service-core/src/runner/teardown.ts index 6e782d15f..77ed3d8a9 100644 --- a/packages/service-core/src/runner/teardown.ts +++ b/packages/service-core/src/runner/teardown.ts @@ -40,22 +40,22 @@ async function terminateSyncRules(storageFactory: storage.BucketStorageFactory, const locks: storage.ReplicationLock[] = []; while (Date.now() - start < 120_000) { let retry = false; - const replicatingSyncRules = await storageFactory.getReplicatingSyncRules(); + const replicatingStreams = await storageFactory.getReplicatingReplicationStreams(); // Lock all the replicating replication streams - for (const replicatingSyncRule of replicatingSyncRules) { - const lock = await replicatingSyncRule.lock(); + for (const replicatingStream of replicatingStreams) { + const lock = await replicatingStream.lock(); locks.push(lock); } - const stoppedSyncRules = await storageFactory.getStoppedSyncRules(); - const combinedSyncRules = [...replicatingSyncRules, ...stoppedSyncRules]; + const stoppedStreams = await storageFactory.getStoppedReplicationStreams(); + const combinedStreams = [...replicatingStreams, ...stoppedStreams]; try { // Clean up any module specific configuration for the replication stream - await moduleManager.tearDown({ syncRules: combinedSyncRules }); + await moduleManager.tearDown({ replicationStreams: combinedStreams, syncRules: combinedStreams }); // Mark the replication stream as terminated - for (let syncRules of combinedSyncRules) { - const syncRulesStorage = storageFactory.getInstance(syncRules); + for (let replicationStream of combinedStreams) { + const syncRulesStorage = storageFactory.getInstance(replicationStream); // The storage will be dropped at the end of the teardown, so we don't need to clear it here await syncRulesStorage.terminate({ clearStorage: false }); } diff --git a/packages/service-core/src/storage/BucketStorageFactory.ts b/packages/service-core/src/storage/BucketStorageFactory.ts index b40f30e33..60ad78686 100644 --- a/packages/service-core/src/storage/BucketStorageFactory.ts +++ b/packages/service-core/src/storage/BucketStorageFactory.ts @@ -2,13 +2,21 @@ import { BaseObserver, logger } from '@powersync/lib-services-framework'; import { PrecompiledSyncConfig, SerializedCompatibilityContext, + SerializedSyncPlanV1, serializeSyncPlan, SqlSyncRules, SyncConfigWithErrors } from '@powersync/service-sync-rules'; import { ReplicationError } from '@powersync/service-types'; import { syncConfigYamlErrorToReplicationError } from '../util/errors.js'; -import { ParseSyncRulesOptions, PersistedSyncRules, PersistedSyncRulesContent } from './PersistedSyncRulesContent.js'; +import { + ParseSyncRulesOptions, + PersistedReplicationStream, + PersistedSyncConfigContent, + PersistedSyncConfigId, + PersistedSyncConfigStatus, + PersistedSyncRules +} from './PersistedSyncRulesContent.js'; import { ReplicationEventPayload } from './ReplicationEventPayload.js'; import { ReplicationLock } from './ReplicationLock.js'; import { ReportStorage } from './ReportStorage.js'; @@ -31,14 +39,14 @@ export abstract class BucketStorageFactory */ async configureSyncRules( options: UpdateSyncRulesOptions - ): Promise<{ updated: boolean; persisted_sync_rules?: PersistedSyncRulesContent; lock?: ReplicationLock }> { - const next = await this.getNextSyncRulesContent(); - const active = await this.getActiveSyncRulesContent(); + ): Promise<{ updated: boolean; persisted_sync_rules?: PersistedReplicationStream; lock?: ReplicationLock }> { + const deploying = await this.getDeployingSyncConfigContent(); + const active = await this.getActiveSyncConfigContent(); - if (next?.sync_rules_content == options.config.yaml) { + if (deploying?.sync_rules_content == options.config.yaml) { logger.info('Sync config unchanged'); return { updated: false }; - } else if (next == null && active?.sync_rules_content == options.config.yaml) { + } else if (deploying == null && active?.sync_rules_content == options.config.yaml) { logger.info('Sync config unchanged'); return { updated: false }; } else { @@ -51,12 +59,15 @@ export abstract class BucketStorageFactory /** * Get a storage instance to query sync data for specific sync config. */ - abstract getInstance(syncRules: PersistedSyncRulesContent, options?: GetIntanceOptions): SyncRulesBucketStorage; + abstract getInstance( + replicationStream: PersistedReplicationStream, + options?: GetIntanceOptions + ): SyncRulesBucketStorage; /** * Deploy new sync config. */ - abstract updateSyncRules(options: UpdateSyncRulesOptions): Promise; + abstract updateSyncRules(options: UpdateSyncRulesOptions): Promise; /** * Indicate that a slot was removed, and we should re-sync by creating @@ -74,37 +85,59 @@ export abstract class BucketStorageFactory * Get the sync config used for querying. */ async getActiveSyncRules(options: ParseSyncRulesOptions): Promise { - const content = await this.getActiveSyncRulesContent(); + const content = await this.getActiveSyncConfigContent(); return content?.parsed(options) ?? null; } /** * Get the sync config used for querying. */ - abstract getActiveSyncRulesContent(): Promise; + abstract getActiveSyncConfigContent(): Promise; /** - * Get the sync config that will be active next once done with initial replicatino. + * Get status for the active sync config. */ - async getNextSyncRules(options: ParseSyncRulesOptions): Promise { - const content = await this.getNextSyncRulesContent(); - return content?.parsed(options) ?? null; + abstract getActiveSyncConfigStatus(): Promise; + + /** + * Get the sync config that is still deploying. + */ + abstract getDeployingSyncConfigContent(): Promise; + + /** + * Get sync configs associated with a replication stream. + */ + abstract getReplicationStreamConfigs(replicationStreamId: number): Promise; + + /** + * Get one exact sync config by persisted config id. + */ + abstract getSyncConfigContent(syncConfigId: PersistedSyncConfigId): Promise; + + /** + * Get per-config statuses associated with a replication stream. + */ + async getReplicationStreamConfigStatuses(replicationStreamId: number): Promise { + return (await this.getReplicationStreamConfigs(replicationStreamId)).map((config) => config.getSyncConfigStatus()); } /** - * Get the sync config that will be active next once done with initial replicatino. + * Get a replication stream by id, regardless of state. + * + * This is the canonical way to obtain a {@link PersistedReplicationStream} for use with + * {@link getInstance} when starting from a {@link PersistedSyncConfigContent}. */ - abstract getNextSyncRulesContent(): Promise; + abstract getReplicationStream(replicationStreamId: number): Promise; /** - * Get all sync config currently replicating. Typically this is the "active" and "next" sync config. + * Get all replication streams currently replicating. */ - abstract getReplicatingSyncRules(): Promise; + abstract getReplicatingReplicationStreams(): Promise; /** - * Get all sync config stopped but not terminated yet. + * Get all replication streams stopped but not terminated yet. */ - abstract getStoppedSyncRules(): Promise; + abstract getStoppedReplicationStreams(): Promise; /** * Get the active storage instance. @@ -181,7 +214,7 @@ export interface SerializedSyncPlan { /** * The serialized plan, from {@link serializeSyncPlan}. */ - plan: unknown; + plan: SerializedSyncPlanV1; compatibility: SerializedCompatibilityContext; /** * Event descriptors are not currently represented in the sync plan because they don't use the sync streams compiler diff --git a/packages/service-core/src/storage/PersistedSyncRulesContent.ts b/packages/service-core/src/storage/PersistedSyncRulesContent.ts index b99328b0f..1b445135c 100644 --- a/packages/service-core/src/storage/PersistedSyncRulesContent.ts +++ b/packages/service-core/src/storage/PersistedSyncRulesContent.ts @@ -16,6 +16,7 @@ import { YamlError } from '@powersync/service-sync-rules'; import * as sqlite from 'node:sqlite'; +import { SyncRuleState } from './BucketStorage.js'; import { SerializedSyncPlan, UpdateSyncRulesOptions } from './BucketStorageFactory.js'; import { ReplicationLock } from './ReplicationLock.js'; import { STORAGE_VERSION_CONFIG, StorageVersionConfig } from './StorageVersionConfig.js'; @@ -24,7 +25,62 @@ export interface ParseSyncRulesOptions { defaultSchema: string; } -export interface PersistedSyncRulesContentData { +export type PersistedSyncConfigId = string; + +export interface PersistedSyncConfigStatus { + readonly id: PersistedSyncConfigId; + readonly replicationStreamId: number; + readonly state: SyncRuleState; + readonly snapshot_done?: boolean; + readonly last_checkpoint_lsn: string | null; + readonly last_fatal_error?: string | null; + readonly last_fatal_error_ts?: Date | null; + readonly last_keepalive_ts?: Date | null; + readonly last_checkpoint_ts?: Date | null; +} + +export interface PersistedReplicationStreamData { + readonly id: number; + readonly slot_name: string; + readonly state: SyncRuleState; + readonly storageVersion: number; + readonly replicationJobId?: string; +} + +export abstract class PersistedReplicationStream implements PersistedReplicationStreamData { + readonly id: number; + readonly replicationJobId: string; + readonly slot_name: string; + readonly state: SyncRuleState; + readonly storageVersion: number; + readonly logger: Logger; + + abstract readonly current_lock: ReplicationLock | null; + + constructor(data: PersistedReplicationStreamData) { + this.id = data.id; + this.replicationJobId = data.replicationJobId ?? String(data.id); + this.slot_name = data.slot_name; + this.state = data.state; + this.storageVersion = data.storageVersion; + this.logger = defaultLogger.child({ prefix: `[${this.slot_name}] ` }); + } + + getStorageConfig(): StorageVersionConfig { + const storageConfig = STORAGE_VERSION_CONFIG[this.storageVersion]; + if (storageConfig == null) { + throw new ServiceError( + ErrorCode.PSYNC_S1005, + `Unsupported storage version ${this.storageVersion} for replication stream ${this.id}` + ); + } + return storageConfig; + } + + abstract lock(): Promise; +} + +export interface PersistedSyncConfigContentData { readonly id: number; readonly sync_rules_content: string; readonly compiled_plan: SerializedSyncPlan | null; @@ -41,28 +97,51 @@ export interface PersistedSyncRulesContentData { readonly last_fatal_error_ts?: Date | null; readonly last_keepalive_ts?: Date | null; readonly last_checkpoint_ts?: Date | null; + readonly state?: SyncRuleState; + readonly syncConfigId?: PersistedSyncConfigId | null; + readonly replicationJobId?: string; } -export abstract class PersistedSyncRulesContent implements PersistedSyncRulesContentData { - readonly id!: number; - readonly sync_rules_content!: string; - readonly compiled_plan!: SerializedSyncPlan | null; - readonly slot_name!: string; - readonly active!: boolean; - readonly storageVersion!: number; +/** + * Immutable sync config content for one sync config inside a replication stream. + * + * This represents the parsed/compiled config plus the per-config status, but + * deliberately does NOT expose stream lifecycle concerns (locking, terminating). + * Use {@link PersistedReplicationStream} for those. + */ +export abstract class PersistedSyncConfigContent implements PersistedSyncConfigContentData { + readonly id: number; + readonly replicationJobId: string; + readonly replicationStreamId: number; + readonly sync_rules_content: string; + readonly compiled_plan: SerializedSyncPlan | null; + readonly slot_name: string; + readonly active: boolean; + readonly state: SyncRuleState; + readonly storageVersion: number; readonly logger: Logger; + readonly syncConfigId: PersistedSyncConfigId | null; - readonly last_checkpoint_lsn!: string | null; + readonly last_checkpoint_lsn: string | null; readonly last_fatal_error?: string | null; readonly last_fatal_error_ts?: Date | null; readonly last_keepalive_ts?: Date | null; readonly last_checkpoint_ts?: Date | null; - abstract readonly current_lock: ReplicationLock | null; - - constructor(data: PersistedSyncRulesContentData) { + constructor(data: PersistedSyncConfigContentData) { Object.assign(this, data); + this.id = data.id; + this.replicationJobId = data.replicationJobId ?? String(data.id); + this.replicationStreamId = data.id; + this.sync_rules_content = data.sync_rules_content; + this.compiled_plan = data.compiled_plan; + this.slot_name = data.slot_name; + this.active = data.active; + this.state = data.state ?? (data.active ? SyncRuleState.ACTIVE : SyncRuleState.PROCESSING); + this.storageVersion = data.storageVersion; + this.syncConfigId = data.syncConfigId ?? null; + this.last_checkpoint_lsn = data.last_checkpoint_lsn; this.logger = defaultLogger.child({ prefix: `[${this.slot_name}] ` }); } @@ -82,9 +161,12 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon return storageConfig; } - parsed(options: ParseSyncRulesOptions): PersistedSyncRules { - let hydrationState: HydrationState; - + /** + * Parse only this config's content into a single {@link SyncConfigWithErrors}. + * + * This does not depend on any other configs in the same replication stream. + */ + protected parseSingleConfig(options: ParseSyncRulesOptions): SyncConfigWithErrors { // Do we have a compiled sync plan? If so, restore from there instead of parsing everything again. let config: SyncConfigWithErrors; if (this.compiled_plan != null) { @@ -127,6 +209,12 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon } else { config = SqlSyncRules.fromYaml(this.sync_rules_content, options); } + return config; + } + + parsed(options: ParseSyncRulesOptions): PersistedSyncRules { + let hydrationState: HydrationState; + const config = this.parseSingleConfig(options); const storageConfig = this.getStorageConfig(); if ( @@ -141,7 +229,7 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon return { id: this.id, slot_name: this.slot_name, - syncConfigWithErrors: config, + syncConfigs: [config], hydrationState, hydratedSyncConfig: () => { return config.config.hydrate({ hydrationState, sqlite: nodeSqlite(sqlite) }); @@ -151,19 +239,30 @@ export abstract class PersistedSyncRulesContent implements PersistedSyncRulesCon asUpdateOptions(options?: Omit): UpdateSyncRulesOptions { // defaultSchema is not relevant for the parsed version here - const parsed = this.parsed({ defaultSchema: 'not_applicable' }); + const parsed = this.parseSingleConfig({ defaultSchema: 'not_applicable' }); return { - config: { yaml: this.sync_rules_content, plan: this.compiled_plan, parsed: parsed.syncConfigWithErrors }, + config: { yaml: this.sync_rules_content, plan: this.compiled_plan, parsed }, ...options }; } - abstract lock(): Promise; + getSyncConfigStatus(): PersistedSyncConfigStatus { + return { + id: this.syncConfigId ?? String(this.id), + replicationStreamId: this.replicationStreamId, + state: this.state, + last_checkpoint_lsn: this.last_checkpoint_lsn, + last_fatal_error: this.last_fatal_error, + last_fatal_error_ts: this.last_fatal_error_ts, + last_keepalive_ts: this.last_keepalive_ts, + last_checkpoint_ts: this.last_checkpoint_ts + }; + } } export interface PersistedSyncRules { readonly id: number; - readonly syncConfigWithErrors: SyncConfigWithErrors; + readonly syncConfigs: SyncConfigWithErrors[]; readonly slot_name: string; /** * For testing only. diff --git a/packages/service-core/test/src/diagnostics.test.ts b/packages/service-core/test/src/diagnostics.test.ts index e72827c51..3e1c88b6f 100644 --- a/packages/service-core/test/src/diagnostics.test.ts +++ b/packages/service-core/test/src/diagnostics.test.ts @@ -13,10 +13,12 @@ bucket_definitions: - SELECT id FROM test_table `; -function makeSyncRulesContent(overrides?: { slot_name?: string }): storage.PersistedSyncRulesContent { +function makeSyncRulesContent(overrides?: { slot_name?: string }): storage.PersistedSyncConfigContent { // We don't implement the entire interface correctly here - just enough to test the diagnostics logic. return { id: 1, + replicationStreamId: 1, + syncConfigId: null, slot_name: overrides?.slot_name ?? 'test_slot', sync_rules_content: MINIMAL_SYNC_RULES, compiled_plan: null, @@ -33,33 +35,29 @@ function makeSyncRulesContent(overrides?: { slot_name?: string }): storage.Persi defaultSchema: 'public' }); return { - syncConfigWithErrors: syncRules + syncConfigs: [syncRules] } as PersistedSyncRules; }, - lock() { - throw new Error('Not implemented in mock'); - }, - current_lock: null, logger: null as any, asUpdateOptions: null as any, getStorageConfig: null as any - } as storage.PersistedSyncRulesContent; + } as storage.PersistedSyncConfigContent; } -function makeBucketStorage() { +function makeSystemStorage() { return { - getInstance() { + async getStatus() { return { - async getStatus() { - return { - snapshot_done: true, - checkpoint_lsn: 'some_lsn', - active: true - }; - } + snapshot_done: true, + checkpoint_lsn: 'some_lsn', + active: true }; } - } as unknown as BucketStorageFactory; + } as storage.SyncRulesBucketStorage; +} + +function makeBucketStorage() { + return {} as unknown as BucketStorageFactory; } function makeRouteAPI(walBudget?: SlotWalBudgetInfo | undefined): RouteAPI { @@ -102,7 +100,14 @@ describe('getSyncRulesStatus WAL budget warnings', () => { safe_wal_size: 4 * GB, max_slot_wal_keep_size: 10 * GB }); - const result = await getSyncRulesStatus(makeBucketStorage(), api, makeSyncRulesContent(), OPTIONS); + const result = await getSyncRulesStatus( + makeBucketStorage(), + api, + makeSyncRulesContent(), + OPTIONS, + null, + makeSystemStorage() + ); const walWarnings = result!.errors.filter((e) => e.message.includes('WAL budget')); expect(walWarnings).toHaveLength(1); expect(walWarnings[0].level).toBe('warning'); @@ -115,7 +120,14 @@ describe('getSyncRulesStatus WAL budget warnings', () => { safe_wal_size: 8 * GB, max_slot_wal_keep_size: 10 * GB }); - const result = await getSyncRulesStatus(makeBucketStorage(), api, makeSyncRulesContent(), OPTIONS); + const result = await getSyncRulesStatus( + makeBucketStorage(), + api, + makeSyncRulesContent(), + OPTIONS, + null, + makeSystemStorage() + ); const walWarnings = result!.errors.filter((e) => e.message.includes('WAL budget')); expect(walWarnings).toHaveLength(0); }); @@ -126,7 +138,14 @@ describe('getSyncRulesStatus WAL budget warnings', () => { safe_wal_size: -2.4 * GB, max_slot_wal_keep_size: 1 * 1024 * 1024 // 1MB }); - const result = await getSyncRulesStatus(makeBucketStorage(), api, makeSyncRulesContent(), OPTIONS); + const result = await getSyncRulesStatus( + makeBucketStorage(), + api, + makeSyncRulesContent(), + OPTIONS, + null, + makeSystemStorage() + ); const walWarnings = result!.errors.filter((e) => e.message.includes('WAL budget')); expect(walWarnings).toHaveLength(1); expect(walWarnings[0].message).toContain('0%'); @@ -137,7 +156,14 @@ describe('getSyncRulesStatus WAL budget warnings', () => { const api = makeRouteAPI({ wal_status: 'lost' }); - const result = await getSyncRulesStatus(makeBucketStorage(), api, makeSyncRulesContent(), OPTIONS); + const result = await getSyncRulesStatus( + makeBucketStorage(), + api, + makeSyncRulesContent(), + OPTIONS, + null, + makeSystemStorage() + ); const walErrors = result!.errors.filter( (e) => e.message.includes('WAL budget') || e.message.includes('PSYNC_S1146') ); @@ -146,10 +172,42 @@ describe('getSyncRulesStatus WAL budget warnings', () => { test('no WAL error when getSlotWalBudget is not defined', async () => { const api = makeRouteAPI(); - const result = await getSyncRulesStatus(makeBucketStorage(), api, makeSyncRulesContent(), OPTIONS); + const result = await getSyncRulesStatus( + makeBucketStorage(), + api, + makeSyncRulesContent(), + OPTIONS, + null, + makeSystemStorage() + ); const walErrors = result!.errors.filter( (e) => e.message.includes('WAL budget') || e.message.includes('PSYNC_S1146') ); expect(walErrors).toHaveLength(0); }); + + test('uses separate sync config status for status-derived diagnostics fields', async () => { + const configStatus: storage.PersistedSyncConfigStatus = { + id: 'config-a', + replicationStreamId: 1, + state: storage.SyncRuleState.ACTIVE, + last_checkpoint_lsn: 'config_lsn', + last_fatal_error: 'config failed', + last_fatal_error_ts: new Date('2026-01-01T00:00:00.000Z'), + last_keepalive_ts: new Date('2026-01-01T00:01:00.000Z'), + last_checkpoint_ts: new Date('2026-01-01T00:02:00.000Z') + }; + const result = await getSyncRulesStatus( + makeBucketStorage(), + makeRouteAPI(), + makeSyncRulesContent(), + OPTIONS, + configStatus, + makeSystemStorage() + ); + + expect(result!.connections[0].last_checkpoint_ts).toBe('2026-01-01T00:02:00.000Z'); + expect(result!.connections[0].last_keepalive_ts).toBe('2026-01-01T00:01:00.000Z'); + expect(result!.errors.some((error) => error.message == 'config failed')).toBe(true); + }); }); diff --git a/packages/service-core/test/src/module-loader.test.ts b/packages/service-core/test/src/module-loader.test.ts index be9cf0230..07b4cc4ce 100644 --- a/packages/service-core/test/src/module-loader.test.ts +++ b/packages/service-core/test/src/module-loader.test.ts @@ -1,4 +1,11 @@ -import { AbstractModule, loadModules, ServiceContextContainer, TearDownOptions } from '@/index.js'; +import { + AbstractModule, + loadModules, + ModuleManager, + ServiceContextContainer, + storage, + TearDownOptions +} from '@/index.js'; import { describe, expect, it } from 'vitest'; interface MockConfig { @@ -27,6 +34,19 @@ class MockPostgresStorageModule extends AbstractModule { async initialize(context: ServiceContextContainer): Promise {} async teardown(options: TearDownOptions): Promise {} } +class RecordingModule extends AbstractModule { + teardownOptions: TearDownOptions | undefined; + + constructor(name: string) { + super({ name }); + } + + async initialize(context: ServiceContextContainer): Promise {} + + async teardown(options: TearDownOptions): Promise { + this.teardownOptions = options; + } +} const mockLoaders = { connection: { mysql: async () => { @@ -99,4 +119,16 @@ describe('module loader', () => { await expect(loadModules(config as any, loaders)).rejects.toThrowError('Failed to load MySQL module'); }); + + it('passes replication streams through teardown options', async () => { + const module = new RecordingModule('RecordingModule'); + const moduleManager = new ModuleManager(); + const replicationStreams = [{ id: 1 }] as storage.PersistedReplicationStream[]; + + moduleManager.register([module]); + await moduleManager.tearDown({ replicationStreams, syncRules: replicationStreams }); + + expect(module.teardownOptions?.replicationStreams).toBe(replicationStreams); + expect(module.teardownOptions?.syncRules).toBe(replicationStreams); + }); }); diff --git a/packages/service-core/test/src/routes/admin.test.ts b/packages/service-core/test/src/routes/admin.test.ts index 8fabe6291..c2aafa458 100644 --- a/packages/service-core/test/src/routes/admin.test.ts +++ b/packages/service-core/test/src/routes/admin.test.ts @@ -1,27 +1,98 @@ -import { BasicRouterRequest, Context, JwtPayload } from '@/index.js'; +import { BasicRouterRequest, Context, JwtPayload, PersistedSyncRules, storage } from '@/index.js'; import { logger } from '@powersync/lib-services-framework'; -import { describe, expect, it } from 'vitest'; -import { validate } from '../../../src/routes/endpoints/admin.js'; +import { SqlSyncRules } from '@powersync/service-sync-rules'; +import { describe, expect, it, vi } from 'vitest'; +import { diagnostics, reprocess, validate } from '../../../src/routes/endpoints/admin.js'; import { mockServiceContext } from './mocks.js'; describe('admin routes', () => { + const request: BasicRouterRequest = { + headers: {}, + hostname: '', + protocol: 'http' + }; + + function makeContext(activeBucketStorage?: any): Context { + const service_context = mockServiceContext(null); + if (activeBucketStorage != null) { + (service_context.storageEngine as any).activeBucketStorage = activeBucketStorage; + } + + return { + logger: logger, + service_context, + token_payload: new JwtPayload({ + sub: '', + exp: 0, + iat: 0 + }) + }; + } + + function makeSyncConfigContent(options: { + id?: number; + syncConfigId?: string; + active?: boolean; + content?: string; + }): storage.PersistedSyncConfigContent { + const id = options.id ?? 1; + const syncConfigId = options.syncConfigId ?? String(id); + const active = options.active ?? true; + const state = active ? storage.SyncRuleState.ACTIVE : storage.SyncRuleState.PROCESSING; + const lastKeepaliveTs = new Date('2026-01-01T00:00:00.000Z'); + const lastCheckpointTs = new Date('2026-01-01T00:00:00.000Z'); + const content = { + id, + replicationStreamId: id, + syncConfigId, + slot_name: `slot_${id}`, + sync_rules_content: + options.content ?? + ` +bucket_definitions: + global: + data: + - SELECT id FROM test +`, + compiled_plan: null, + active, + state, + storageVersion: storage.LEGACY_STORAGE_VERSION, + last_checkpoint_lsn: null, + last_fatal_error: null, + last_fatal_error_ts: null, + last_keepalive_ts: lastKeepaliveTs, + last_checkpoint_ts: lastCheckpointTs, + parsed(options?: any) { + const syncRules = SqlSyncRules.fromYaml(content.sync_rules_content, { + ...options, + defaultSchema: 'public' + }); + return { + syncConfigs: [syncRules] + } as PersistedSyncRules; + }, + asUpdateOptions: vi.fn(), + getStorageConfig: vi.fn(), + getSyncConfigStatus() { + return { + id: syncConfigId, + replicationStreamId: id, + state, + last_checkpoint_lsn: null, + last_fatal_error: null, + last_fatal_error_ts: null, + last_keepalive_ts: lastKeepaliveTs, + last_checkpoint_ts: lastCheckpointTs + }; + } + }; + return content as unknown as storage.PersistedSyncConfigContent; + } + describe('validate', () => { it('reports errors with source location', async () => { - const context: Context = { - logger: logger, - service_context: mockServiceContext(null), - token_payload: new JwtPayload({ - sub: '', - exp: 0, - iat: 0 - }) - }; - - const request: BasicRouterRequest = { - headers: {}, - hostname: '', - protocol: 'http' - }; + const context = makeContext(); const response = await validate.handler({ context, @@ -45,4 +116,92 @@ bucket_definitions: ]); }); }); + + describe('diagnostics', () => { + it('returns deploying config status', async () => { + const active = makeSyncConfigContent({ id: 1, syncConfigId: 'active-config' }); + const deploying = makeSyncConfigContent({ id: 2, syncConfigId: 'deploying-config', active: false }); + const getInstance = vi.fn(() => ({ + async getStatus() { + return { + active: true, + snapshot_done: false, + checkpoint_lsn: null + }; + } + })); + const activeBucketStorage = { + getActiveSyncConfigContent: vi.fn(async () => active), + getActiveSyncConfigStatus: vi.fn(async () => active.getSyncConfigStatus()), + getActiveStorage: vi.fn(async () => getInstance()), + getDeployingSyncConfigContent: vi.fn(async () => deploying), + getReplicationStream: vi.fn(async (id: number) => ({ id, slot_name: `slot_${id}` })), + getInstance + }; + + const response = await diagnostics.handler({ + context: makeContext(activeBucketStorage), + params: {}, + request + }); + + expect(response.deploying_sync_rules?.connections[0].slot_name).toBe('slot_2'); + expect(response.active_sync_rules?.connections[0].slot_name).toBe('slot_1'); + }); + }); + + describe('reprocess', () => { + it('reprocesses the active sync config', async () => { + const active = makeSyncConfigContent({ id: 7, syncConfigId: 'active-config' }); + const updateSyncRules = vi.fn(async () => ({ + id: 8, + slot_name: 'new_slot', + state: storage.SyncRuleState.PROCESSING, + storageVersion: storage.LEGACY_STORAGE_VERSION, + replicationJobId: '8', + current_lock: null + })); + const activeBucketStorage = { + getDeployingSyncConfigContent: vi.fn(async () => null), + getActiveSyncConfigContent: vi.fn(async () => active), + getSyncConfigContent: vi.fn(), + updateSyncRules + }; + + const response = await reprocess.handler({ + context: makeContext(activeBucketStorage), + params: {}, + request + }); + + expect(activeBucketStorage.getActiveSyncConfigContent).toHaveBeenCalledTimes(1); + expect(activeBucketStorage.getSyncConfigContent).not.toHaveBeenCalled(); + expect(updateSyncRules).toHaveBeenCalledTimes(1); + expect(response.connections[0].slot_name).toBe('new_slot'); + }); + + it('rejects reprocess while a sync config is deploying', async () => { + const activeBucketStorage = { + getDeployingSyncConfigContent: vi.fn(async () => makeSyncConfigContent({ id: 2, active: false })), + getActiveSyncConfigContent: vi.fn(), + updateSyncRules: vi.fn() + }; + + await expect( + reprocess.handler({ + context: makeContext(activeBucketStorage), + params: {}, + request + }) + ).rejects.toMatchObject({ + errorData: { + status: 409, + code: 'PSYNC_S4106', + description: 'Busy processing sync config - cannot reprocess' + } + }); + expect(activeBucketStorage.getActiveSyncConfigContent).not.toHaveBeenCalled(); + expect(activeBucketStorage.updateSyncRules).not.toHaveBeenCalled(); + }); + }); }); diff --git a/packages/service-errors/src/codes.ts b/packages/service-errors/src/codes.ts index 6ad3d38d7..e423dd25f 100644 --- a/packages/service-errors/src/codes.ts +++ b/packages/service-errors/src/codes.ts @@ -534,5 +534,10 @@ export enum ErrorCode { * * When a sync config is specified in the service config, the dynamic sync config API is disabled. */ - PSYNC_S4105 = 'PSYNC_S4105' + PSYNC_S4105 = 'PSYNC_S4105', + + /** + * Sync config reprocess is blocked because another sync config is deploying. + */ + PSYNC_S4106 = 'PSYNC_S4106' } diff --git a/packages/sync-rules/src/index.ts b/packages/sync-rules/src/index.ts index 22b916420..fe28562cc 100644 --- a/packages/sync-rules/src/index.ts +++ b/packages/sync-rules/src/index.ts @@ -34,7 +34,10 @@ export * from './types/time.js'; export * from './utils.js'; export * from './compiler/compiler.js'; -export { Database, SQLite, Statement, nodeSqlite } from './sync_plan/engine/sqlite.js'; +export { HashMap, HashSet } from './compiler/equality.js'; +export { javaScriptExpressionEngine } from './sync_plan/engine/javascript.js'; +export { Database, SQLite, Statement, nodeSqlite, sqliteExpressionEngine } from './sync_plan/engine/sqlite.js'; export { PrecompiledSyncConfig } from './sync_plan/evaluator/index.js'; export * from './sync_plan/plan.js'; -export { deserializeSyncPlan, serializeSyncPlan } from './sync_plan/serialize.js'; +export * from './sync_plan/plan_equality_serialized.js'; +export * from './sync_plan/serialize.js'; diff --git a/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts b/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts new file mode 100644 index 000000000..55bdb997b --- /dev/null +++ b/packages/sync-rules/src/sync_plan/plan_equality_serialized.ts @@ -0,0 +1,47 @@ +import { Equality } from '../compiler/equality.js'; +import { + SerializedBucketDataSource, + SerializedDataSource, + SerializedParameterIndexLookupCreator +} from './serialize.js'; + +export interface SerializedBucketDataSourceWithDataSources { + bucket: SerializedBucketDataSource; + dataSources: readonly SerializedDataSource[]; +} + +export const serializedStreamDataSourceEquality = jsonEquality(); + +export const serializedStreamParameterIndexLookupCreatorEquality = + jsonEquality(); + +export const serializedStreamBucketDataSourceEquality: Equality = { + hash(hasher, value) { + hasher.addString(JSON.stringify(bucketIdentity(value))); + }, + equals(a, b) { + return a === b || JSON.stringify(bucketIdentity(a)) == JSON.stringify(bucketIdentity(b)); + } +}; + +function bucketIdentity(value: SerializedBucketDataSourceWithDataSources) { + const { bucket, dataSources } = value; + + return { + hash: bucket.hash, + uniqueName: bucket.uniqueName, + // Sort so that the order of sources does not affect equality, as long as the same data sources are included. + sources: bucket.sources.map((index) => JSON.stringify(dataSources[index])).sort() + }; +} + +function jsonEquality(): Equality { + return { + hash(hasher, value) { + hasher.addString(JSON.stringify(value)); + }, + equals(a, b) { + return a === b || JSON.stringify(a) == JSON.stringify(b); + } + }; +} diff --git a/packages/sync-rules/src/sync_plan/serialize.ts b/packages/sync-rules/src/sync_plan/serialize.ts index cc3f6eb93..f6806e95e 100644 --- a/packages/sync-rules/src/sync_plan/serialize.ts +++ b/packages/sync-rules/src/sync_plan/serialize.ts @@ -332,7 +332,7 @@ export function deserializeSyncPlan(serialized: unknown): SyncPlan { }; } -interface SerializedSyncPlanV1 { +export interface SerializedSyncPlanV1 { version: number; dataSources: SerializedDataSource[]; buckets: SerializedBucketDataSource[]; @@ -340,32 +340,32 @@ interface SerializedSyncPlanV1 { streams: SerializedStream[]; } -interface SerializedBucketDataSource { +export interface SerializedBucketDataSource { hash: number; uniqueName: string; sources: number[]; } -interface SerializedTablePattern { +export interface SerializedTablePattern { connection: string | null; schema: string | null; table: string; } -interface SerializedTableProcessorTableValuedFunctionOutput { +export interface SerializedTableProcessorTableValuedFunctionOutput { function: number; outputName: string; } -type SerializedTableProcessorData = ColumnSqlParameterValue | SerializedTableProcessorTableValuedFunctionOutput; +export type SerializedTableProcessorData = ColumnSqlParameterValue | SerializedTableProcessorTableValuedFunctionOutput; -interface SerializedPartitionKey { +export interface SerializedPartitionKey { expr: SqlExpression; } -type SerializedColumnSource = 'star' | { expr: SqlExpression; alias: string }; +export type SerializedColumnSource = 'star' | { expr: SqlExpression; alias: string }; -interface SerializedDataSource { +export interface SerializedDataSource { table: SerializedTablePattern; outputTableName?: string; hash: number; @@ -375,7 +375,7 @@ interface SerializedDataSource { partitionBy: SerializedPartitionKey[]; } -interface SerializedParameterIndexLookupCreator { +export interface SerializedParameterIndexLookupCreator { table: SerializedTablePattern; hash: number; lookupScope: ParameterLookupDefinitionId; @@ -385,19 +385,19 @@ interface SerializedParameterIndexLookupCreator { partitionBy: SerializedPartitionKey[]; } -interface SerializedStream { +export interface SerializedStream { stream: StreamOptions; queriers: SerializedStreamQuerier[]; } -interface SerializedStreamQuerier { +export interface SerializedStreamQuerier { requestFilters: SqlExpression[]; lookupStages: SerializedExpandingLookup[][]; bucket: number; sourceInstantiation: SerializedParameterValue[]; } -type SerializedExpandingLookup = +export type SerializedExpandingLookup = | { type: 'parameter'; lookup: number; @@ -411,12 +411,12 @@ type SerializedExpandingLookup = filters: SqlExpression[]; }; -interface LookupReference { +export interface LookupReference { stageId: number; idInStage: number; } -type SerializedParameterValue = +export type SerializedParameterValue = | { type: 'request'; expr: SqlExpression } | { type: 'lookup'; lookup: LookupReference; resultIndex: number } | { type: 'intersection'; values: SerializedParameterValue[] }; diff --git a/packages/sync-rules/test/src/sync_plan/evaluator/bucket_data_source_equality.test.ts b/packages/sync-rules/test/src/sync_plan/evaluator/bucket_data_source_equality.test.ts new file mode 100644 index 000000000..e36fa7525 --- /dev/null +++ b/packages/sync-rules/test/src/sync_plan/evaluator/bucket_data_source_equality.test.ts @@ -0,0 +1,255 @@ +import { describe, expect } from 'vitest'; +import { Equality, StableHasher } from '../../../../src/compiler/equality.js'; +import { + BucketDataSource, + SerializedBucketDataSourceWithDataSources, + serializedStreamBucketDataSourceEquality, + serializeSyncPlan, + SyncConfig +} from '../../../../src/index.js'; +import { compileToSyncPlanWithoutErrors } from '../../compiler/utils.js'; +import { syncTest } from './utils.js'; + +describe('prepared bucket data source equality', () => { + syncTest('matches equivalent prepared bucket sources from different plans', ({ sync }) => { + const firstYaml = ` +config: + edition: 3 + +streams: + stream: + query: SELECT id, owner_id FROM todos WHERE owner_id = auth.user_id() +`; + const secondYaml = ` +config: + edition: 3 + +streams: + stream: + query: SELECT id, owner_id FROM todos WHERE owner_id = auth.user_id() +`; + const first = firstBucketSource(sync.prepareWithoutHydration(firstYaml)); + const second = firstBucketSource(sync.prepareWithoutHydration(secondYaml)); + + expectSerializedBucketSources(firstYaml, secondYaml, true); + }); + + syncTest('does not match when output columns differ', ({ sync }) => { + const firstYaml = ` +config: + edition: 3 + +streams: + stream: + query: SELECT id, owner_id FROM todos WHERE owner_id = auth.user_id() +`; + const secondYaml = ` +config: + edition: 3 + +streams: + stream: + query: SELECT id, title FROM todos WHERE owner_id = auth.user_id() +`; + const first = firstBucketSource(sync.prepareWithoutHydration(firstYaml)); + const second = firstBucketSource(sync.prepareWithoutHydration(secondYaml)); + + expectSerializedBucketSources(firstYaml, secondYaml, false); + }); + + syncTest('does not match when partitioning differs', ({ sync }) => { + const firstYaml = ` +config: + edition: 3 + +streams: + stream: + query: SELECT * FROM todos WHERE owner_id = auth.user_id() +`; + const secondYaml = ` +config: + edition: 3 + +streams: + stream: + query: SELECT * FROM todos WHERE project_id = auth.user_id() +`; + const first = firstBucketSource(sync.prepareWithoutHydration(firstYaml)); + const second = firstBucketSource(sync.prepareWithoutHydration(secondYaml)); + + expectSerializedBucketSources(firstYaml, secondYaml, false); + }); + + syncTest('does not match equivalent bucket sources with different stream names', ({ sync }) => { + const firstYaml = ` +config: + edition: 3 + +streams: + first_stream: + query: SELECT id, owner_id FROM todos WHERE owner_id = auth.user_id() +`; + const secondYaml = ` +config: + edition: 3 + +streams: + second_stream: + query: SELECT id, owner_id FROM todos WHERE owner_id = auth.user_id() +`; + const first = firstBucketSource(sync.prepareWithoutHydration(firstYaml)); + const second = firstBucketSource(sync.prepareWithoutHydration(secondYaml)); + + expect(first.uniqueName).not.toEqual(second.uniqueName); + expectSerializedBucketSources(firstYaml, secondYaml, false, true); + }); + + syncTest('matches when subqueries differ but the data source does not', ({ sync }) => { + const firstYaml = ` +config: + edition: 3 + +streams: + stream: + query: | + SELECT * FROM projects + WHERE org_id IN ( + SELECT org_id FROM memberships WHERE user_id = auth.user_id() + ) +`; + const secondYaml = ` +config: + edition: 3 + +streams: + stream: + query: | + SELECT * FROM projects + WHERE org_id IN ( + SELECT id FROM organizations WHERE owner_id = auth.user_id() + ) +`; + const first = firstBucketSource(sync.prepareWithoutHydration(firstYaml)); + const second = firstBucketSource(sync.prepareWithoutHydration(secondYaml)); + + expectSerializedBucketSources(firstYaml, secondYaml, true); + }); + + syncTest('matches when only bucket input parameters differ', ({ sync }) => { + const firstYaml = ` +config: + edition: 3 + +streams: + stream: + query: SELECT * FROM todos WHERE owner_id = auth.user_id() +`; + const secondYaml = ` +config: + edition: 3 + +streams: + stream: + query: SELECT * FROM todos WHERE owner_id = auth.parameter('user_id') +`; + const first = firstBucketSource(sync.prepareWithoutHydration(firstYaml)); + const second = firstBucketSource(sync.prepareWithoutHydration(secondYaml)); + + expectSerializedBucketSources(firstYaml, secondYaml, true); + }); + + syncTest('matches buckets with the same data sources in a different order', ({ sync }) => { + const firstYaml = ` +config: + edition: 3 + +streams: + stream: + queries: + - SELECT * FROM products + - SELECT * FROM stores +`; + const secondYaml = ` +config: + edition: 3 + +streams: + stream: + queries: + - SELECT * FROM stores + - SELECT * FROM products +`; + const first = firstBucketSource(sync.prepareWithoutHydration(firstYaml)); + const second = firstBucketSource(sync.prepareWithoutHydration(secondYaml)); + + expectSerializedBucketSources(firstYaml, secondYaml, true); + }); + + syncTest('compares table-valued function output expressions by their bindings', ({ sync }) => { + const firstYaml = ` +config: + edition: 3 + +streams: + stream: + query: | + SELECT customers.id as id + FROM customers, json_each(customers.active_regions) AS region + WHERE region.value < 'm' +`; + const secondYaml = ` +config: + edition: 3 + +streams: + stream: + query: | + SELECT customers.id as id + FROM customers, json_each(customers.active_regions) AS region + WHERE region.value < 'm' +`; + const first = firstBucketSource(sync.prepareWithoutHydration(firstYaml)); + const second = firstBucketSource(sync.prepareWithoutHydration(secondYaml)); + + expectSerializedBucketSources(firstYaml, secondYaml, true); + }); +}); + +function firstBucketSource(config: SyncConfig): BucketDataSource { + return config.bucketDataSources[0]; +} + +function expectSerializedBucketSources( + firstYaml: string, + secondYaml: string, + equal: boolean, + expectDifferentUniqueNames = false +) { + const first = firstSerializedBucketSource(firstYaml); + const second = firstSerializedBucketSource(secondYaml); + + if (expectDifferentUniqueNames) { + expect(first.bucket.uniqueName).not.toEqual(second.bucket.uniqueName); + } + + expect(serializedStreamBucketDataSourceEquality.equals(first, second)).toBe(equal); + if (equal) { + expect(hashWith(serializedStreamBucketDataSourceEquality, first)).toEqual( + hashWith(serializedStreamBucketDataSourceEquality, second) + ); + } +} + +function firstSerializedBucketSource(yaml: string): SerializedBucketDataSourceWithDataSources { + const plan = serializeSyncPlan(compileToSyncPlanWithoutErrors(yaml)); + return { + bucket: plan.buckets[0], + dataSources: plan.dataSources + }; +} + +function hashWith(equality: Equality, value: T): number { + const hasher = new StableHasher(); + equality.hash(hasher, value); + return hasher.buildHashCode(); +}