diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3f43cfe9..9003d021 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -17,6 +17,8 @@ /src/commands/analysis/ @celonis/process-analytics /src/commands/cpm4/ @celonis/cpm4 /src/commands/data-pipeline/ @Dusan-r @IvanGandacov @EktaCelonis @gorasoCelonis +/src/commands/data-model-migration/ @celonis/studio-platform +/tests/commands/data-model-migration/ @celonis/studio-platform /src/commands/studio/ @celonis/astro @celonis/studio-platform /tests/commands/studio/ @celonis/astro @celonis/studio-platform /package.json @celonis/studio-platform @aocelo @siavash-celonis diff --git a/docs/command-graph.html b/docs/command-graph.html index ed95d759..2a0d2c5b 100644 --- a/docs/command-graph.html +++ b/docs/command-graph.html @@ -211,6 +211,9 @@ { id: "export_data_pool", label: "data-pool", group: "command", path: "export data-pool", description: "Command to export a data pool", options: ["-p, --profile ", "--id ", "--outputToJsonFile", "-h, --help"] }, + { id: "export_data_model", label: "data-model", group: "command", path: "export data-model", + description: "Export a data model with tables, foreign keys, and process configurations", + options: ["-p, --profile ", "--poolId ", "--dataModelId ", "--outputToJsonFile", "-h, --help"] }, // import { id: "import_action_flows", label: "action-flows", group: "command", path: "import action-flows", @@ -247,6 +250,9 @@ description: "Command to push a data pool", options: ["-p, --profile ", "-f, --file ", "-h, --help"] }, { id: "push_data_pools", label: "data-pools", group: "command", path: "push data-pools", description: "Command to push data pools", options: ["-p, --profile ", "-h, --help"] }, + { id: "push_semantic_model", label: "semantic-model", group: "command", path: "push semantic-model", + description: "Convert a data model into semantic entities and push them into a pig package", + options: ["-p, --profile ", "--poolId ", "--dataModelId ", "--package ", "--schema ", "--namespace ", "-f, --fromFile ", "--dryRun", "--outputToJsonFile", "-h, --help"] }, { id: "push_asset", label: "asset", group: "command", path: "push asset", description: "Command to push an asset to Studio", options: ["-p, --profile ", "-f, --file ", "--package ", "-h, --help"] }, { id: "push_assets", label: "assets", group: "command", path: "push assets", @@ -479,7 +485,7 @@ ["area_analyze","analyze_action_flows"], - ["area_export","export_action_flows"],["area_export","export_data_pool"], + ["area_export","export_action_flows"],["area_export","export_data_pool"],["area_export","export_data_model"], ["area_import","import_action_flows"],["area_import","import_data_pools"], @@ -487,7 +493,7 @@ ["area_pull","pull_asset"],["area_pull","pull_package"],["area_pull","pull_view_bookmarks"], ["area_push","push_skill"],["area_push","push_bookmarks"],["area_push","push_ctp"], - ["area_push","push_data_pool"],["area_push","push_data_pools"],["area_push","push_asset"], + ["area_push","push_data_pool"],["area_push","push_data_pools"],["area_push","push_semantic_model"],["area_push","push_asset"], ["area_push","push_assets"],["area_push","push_package"],["area_push","push_packages"], ["area_push","push_widget"],["area_push","push_view_bookmarks"], diff --git a/docs/index.md b/docs/index.md index 1e47cd50..abac28ec 100644 --- a/docs/index.md +++ b/docs/index.md @@ -53,6 +53,7 @@ content-cli pull package -h - [Config Commands](user-guide/config-commands.md) -- Batch export/import, variables, nodes, diffs, and dependencies - [Deployment Commands](user-guide/deployment-commands.md) -- Create, history, active deployments, deployables, and targets - [Data Pool Commands](user-guide/data-pool-commands.md) -- Export/import data pools and connection management + - [Data Model Migration Commands](user-guide/data-model-migration-commands.md) -- Export data models and push semantic entities to pig packages - [Action Flow Commands](user-guide/action-flow-commands.md) -- Analyze and export action flows - **Development** - [Architecture](internal-architecture.md) -- Internal architecture and inner workings diff --git a/docs/user-guide/data-model-migration-commands.md b/docs/user-guide/data-model-migration-commands.md new file mode 100644 index 00000000..cc02fe9e --- /dev/null +++ b/docs/user-guide/data-model-migration-commands.md @@ -0,0 +1,79 @@ +# Data Model Migration Commands + +These commands export a Data Integration data model from cloud-data-integration and convert it into semantic entity nodes in a target OCDM package via the Pacman staging-node API (same path as `config nodes create`). + +Supported mappings: + +| Data Integration | Semantic entity | +|---|---| +| Table | Object (with data binding) | +| Process configuration (event log) | Event source (with data binding) | +| Classic foreign key | Relationship | +| Data model | Perspective | + +**Object links** (`signal-links` in cloud-data-integration) are **not** supported. The converter only reads classic `foreignKeys[]` from the `/transport` export. + +## Export Data Model + +Downloads the full data model transport (tables, columns, foreign keys, process configurations): + +``` +content-cli export data-model --poolId --dataModelId --profile [--outputToJsonFile] +``` + +Example: + +``` +content-cli export data-model --poolId 80a1389d-50c5-4976-ad6e-fb5b7a2b5517 --dataModelId 1b9b368b-e0df-4e74-99e8-59e2febe9687 --profile local --outputToJsonFile +``` + +## Push Semantic Model + +Converts the data model and pushes semantic entity nodes into a target OCDM package: + +``` +content-cli push semantic-model \ + --poolId \ + --dataModelId \ + --package \ + --profile \ + [--schema ] \ + [--namespace ] \ + [--fromFile ] \ + [--dryRun] \ + [--outputToJsonFile] +``` + +Options: + +- `--schema`: Physical lake schema used in data bindings. When omitted, the CLI derives `datapipelines__draft` (hyphens in the pool id become underscores). +- `--fromFile`: Skip download and convert a previously exported transport JSON file. +- `--dryRun`: Convert only; print or write the Pacman node payloads without calling the staging-node API. +- `--namespace`: Optional namespace for data bindings (entity references use the package `local` namespace by default). + +Push order: objects → event sources → relationships → perspective. + +Each entity is created as a Pacman staging node with types `SEMANTIC_OBJECT_TYPE`, `SEMANTIC_EVENT_SOURCE_TYPE`, `SEMANTIC_RELATIONSHIP_TYPE`, and `SEMANTIC_PERSPECTIVE_TYPE`. + +After pushing, validate and version the authored nodes with the standard config commands: + +``` +content-cli config package validate --packageKey --nodeKeys --layers SCHEMA BUSINESS +content-cli config versions create --packageKey --nodeFilterKeys --versionBumpOption PATCH --summaryOfChanges "..." +``` + +Example dry run: + +``` +content-cli push semantic-model \ + --poolId 80a1389d-50c5-4976-ad6e-fb5b7a2b5517 \ + --dataModelId 1b9b368b-e0df-4e74-99e8-59e2febe9687 \ + --package my-context-model \ + --profile local \ + --dryRun +``` + +## Authentication + +- **Download** uses the existing `integration.data-pools` OAuth scope (same as other data pool commands). +- **Push** uses the existing `package-manager` OAuth scope via `/pacman/api/core/staging/packages/{packageKey}/nodes`. Ensure the profile has edit access to the target package. diff --git a/mkdocs.yaml b/mkdocs.yaml index 4648ae2f..779aa9d3 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -18,6 +18,7 @@ nav: - Deployment Commands: './user-guide/deployment-commands.md' - Asset Registry Commands: './user-guide/asset-registry-commands.md' - Data Pool Commands: './user-guide/data-pool-commands.md' + - Data Model Migration Commands: './user-guide/data-model-migration-commands.md' - Action Flow Commands: './user-guide/action-flow-commands.md' - Development: - Architecture: './internal-architecture.md' diff --git a/src/commands/data-model-migration/api/data-model-api.ts b/src/commands/data-model-migration/api/data-model-api.ts new file mode 100644 index 00000000..2ac2d5f8 --- /dev/null +++ b/src/commands/data-model-migration/api/data-model-api.ts @@ -0,0 +1,23 @@ +import { Context } from "../../../core/command/cli-context"; +import { FatalError } from "../../../core/utils/logger"; +import { HttpClient } from "../../../core/http/http-client"; +import { DataModelTransport } from "../interfaces/data-model-transport.interfaces"; + +export class DataModelApi { + + private httpClient: () => HttpClient; + + constructor(context: Context) { + this.httpClient = () => context.httpClient; + } + + /** Fetches the full data model transport including columns from cloud-data-integration. */ + public async findOneTransport(poolId: string, dataModelId: string, includeColumns = true): Promise { + const query = includeColumns ? "?includeColumns=true" : ""; + return this.httpClient() + .get(`/integration/api/pools/${poolId}/data-models/${dataModelId}/transport${query}`) + .catch((error) => { + throw new FatalError(`Data model export failed for pool ${poolId}, data model ${dataModelId}: ${error}`); + }); + } +} diff --git a/src/commands/data-model-migration/constants/semantic-node.constants.ts b/src/commands/data-model-migration/constants/semantic-node.constants.ts new file mode 100644 index 00000000..d2d3898e --- /dev/null +++ b/src/commands/data-model-migration/constants/semantic-node.constants.ts @@ -0,0 +1,10 @@ +/** Pacman node type identifiers for context-model semantic assets. */ +export const SEMANTIC_NODE_TYPES = { + OBJECT: "SEMANTIC_OBJECT_TYPE", + EVENT_SOURCE: "SEMANTIC_EVENT_SOURCE_TYPE", + RELATIONSHIP: "SEMANTIC_RELATIONSHIP_TYPE", + PERSPECTIVE: "SEMANTIC_PERSPECTIVE_TYPE", +} as const; + +/** Asset-registry schema version for semantic entity node types. */ +export const SEMANTIC_SCHEMA_VERSION = 1; diff --git a/src/commands/data-model-migration/data-model-migration-command.service.ts b/src/commands/data-model-migration/data-model-migration-command.service.ts new file mode 100644 index 00000000..e4400d67 --- /dev/null +++ b/src/commands/data-model-migration/data-model-migration-command.service.ts @@ -0,0 +1,28 @@ +import { Context } from "../../core/command/cli-context"; +import { DataModelMigrationService } from "./service/data-model-migration.service"; + +export class DataModelMigrationCommandService { + + private migrationService: DataModelMigrationService; + + constructor(context: Context) { + this.migrationService = new DataModelMigrationService(context); + } + + public async exportDataModel(poolId: string, dataModelId: string, outputToJsonFile: boolean): Promise { + await this.migrationService.exportDataModel(poolId, dataModelId, outputToJsonFile); + } + + public async pushSemanticModel(options: { + poolId: string; + dataModelId: string; + packageKey: string; + schema?: string; + namespace?: string; + fromFile?: string; + dryRun?: boolean; + outputToJsonFile?: boolean; + }): Promise { + await this.migrationService.pushSemanticModel(options); + } +} diff --git a/src/commands/data-model-migration/data-model-migration.commands.ts b/src/commands/data-model-migration/data-model-migration.commands.ts new file mode 100644 index 00000000..7d5e1958 --- /dev/null +++ b/src/commands/data-model-migration/data-model-migration.commands.ts @@ -0,0 +1,55 @@ +import { Context } from "../../core/command/cli-context"; +import { Configurator } from "../../core/command/module-handler"; +import { Command, OptionValues } from "commander"; +import { DataModelMigrationCommandService } from "./data-model-migration-command.service"; + +export class DataModelMigrationCommands { + + public register(_context: Context, configurator: Configurator): void { + configurator.command("export") + .command("data-model") + .description("Export a data model with tables, foreign keys, and process configurations") + .requiredOption("--poolId ", "ID of the data pool") + .requiredOption("--dataModelId ", "ID of the data model") + .option("--outputToJsonFile", "Write the exported data model to a JSON file") + .action(this.exportDataModel); + + configurator.command("push") + .command("semantic-model") + .description("Convert a data model into semantic entities and push them into a pig package") + .requiredOption("--package ", "Target pig package key") + .option("--poolId ", "ID of the data pool (required unless --fromFile is set)") + .option("--dataModelId ", "ID of the data model (required unless --fromFile is set)") + .option("--schema ", "Physical lake schema for data bindings (overrides pool-derived default)") + .option("--namespace ", "Namespace for created semantic entities") + .option("-f, --fromFile ", "Use a previously exported data model transport JSON file") + .option("--dryRun", "Convert only; print or write node payloads without pushing to Pacman") + .option("--outputToJsonFile", "With --dryRun, write conversion output to a JSON file") + .action(this.pushSemanticModel); + } + + private async exportDataModel(context: Context, _command: Command, options: OptionValues): Promise { + await new DataModelMigrationCommandService(context).exportDataModel( + options.poolId, + options.dataModelId, + !!options.outputToJsonFile + ); + } + + private async pushSemanticModel(context: Context, _command: Command, options: OptionValues): Promise { + if (!options.fromFile && (!options.poolId || !options.dataModelId)) { + throw new Error("Either --fromFile or both --poolId and --dataModelId are required"); + } + + await new DataModelMigrationCommandService(context).pushSemanticModel({ + poolId: options.poolId, + dataModelId: options.dataModelId, + packageKey: options.package, + schema: options.schema, + namespace: options.namespace, + fromFile: options.fromFile, + dryRun: !!options.dryRun, + outputToJsonFile: !!options.outputToJsonFile, + }); + } +} diff --git a/src/commands/data-model-migration/interfaces/conversion-result.interfaces.ts b/src/commands/data-model-migration/interfaces/conversion-result.interfaces.ts new file mode 100644 index 00000000..48977ea7 --- /dev/null +++ b/src/commands/data-model-migration/interfaces/conversion-result.interfaces.ts @@ -0,0 +1,15 @@ +import { SaveNodeTransport } from "../../configuration-management/interfaces/node.interfaces"; + +export interface ConversionResult { + objects: SaveNodeTransport[]; + eventSources: SaveNodeTransport[]; + relationships: SaveNodeTransport[]; + perspective: SaveNodeTransport; +} + +export interface ConversionOptions { + poolId: string; + bindingSchema: string; + packageKey: string; + namespace?: string; +} diff --git a/src/commands/data-model-migration/interfaces/data-model-transport.interfaces.ts b/src/commands/data-model-migration/interfaces/data-model-transport.interfaces.ts new file mode 100644 index 00000000..87c65958 --- /dev/null +++ b/src/commands/data-model-migration/interfaces/data-model-transport.interfaces.ts @@ -0,0 +1,59 @@ +export type ColumnType = "INTEGER" | "DATE" | "TIME" | "DATETIME" | "FLOAT" | "BOOLEAN" | "STRING"; + +export type DataModelType = "CASE_CENTRIC" | "OBJECT_CENTRIC"; + +export interface DataModelColumnTransport { + name: string; + type: ColumnType; + primaryKey?: boolean; +} + +export interface DataModelTableTransport { + id: string; + name: string; + alias?: string; + aliasOrName?: string; + dataModelId?: string; + dataSourceId?: string; + primaryKeys?: string[]; + columns?: DataModelColumnTransport[]; +} + +export interface DataModelForeignKeyColumnTransport { + id?: string; + sourceColumnName: string; + targetColumnName: string; +} + +export interface DataModelForeignKeyTransport { + id: string; + dataModelId?: string; + sourceTableId: string; + targetTableId: string; + columns: DataModelForeignKeyColumnTransport[]; +} + +export interface DataModelConfigurationTransport { + id?: string; + dataModelId?: string; + activityTableId: string; + caseTableId?: string; + caseIdColumn: string; + activityColumn: string; + timestampColumn: string; + endTimestampColumn?: string; + sortingColumn?: string; + costColumn?: string; + userColumn?: string; + defaultConfiguration?: boolean; +} + +export interface DataModelTransport { + id: string; + name: string; + poolId?: string; + dataModelType?: DataModelType; + tables: DataModelTableTransport[]; + foreignKeys?: DataModelForeignKeyTransport[]; + processConfigurations?: DataModelConfigurationTransport[]; +} diff --git a/src/commands/data-model-migration/interfaces/semantic-entity.interfaces.ts b/src/commands/data-model-migration/interfaces/semantic-entity.interfaces.ts new file mode 100644 index 00000000..8b0d2a39 --- /dev/null +++ b/src/commands/data-model-migration/interfaces/semantic-entity.interfaces.ts @@ -0,0 +1,81 @@ +export type AttributeDataType = + | "STRING" + | "INTEGER" + | "LONG" + | "DOUBLE" + | "BOOLEAN" + | "TIMESTAMP" + | "DATE"; + +export type PigEntityReferenceType = "OBJECT" | "EVENT_SOURCE" | "RELATIONSHIP"; + +export type RelationshipType = "TYPE_TO_TYPE" | "INSTANCE_TO_INSTANCE"; + +export type Cardinality = "ONE_TO_ONE" | "ONE_TO_MANY" | "MANY_TO_ONE" | "MANY_TO_MANY"; + +export type PerspectiveType = "LIVE" | "CACHED"; + +export interface OntologyAttribute { + id: string; + dataType: AttributeDataType; + required?: boolean; + description?: string; +} + +export interface MappingColumn { + sourceColumn: string; + targetColumn: string; +} + +export interface Binding { + name: string; + namespace?: string; + schema: string; + table: string; + mappingColumns: MappingColumn[]; +} + +export interface SemanticObjectConfiguration { + active: boolean; + attributes: OntologyAttribute[]; + bindings: Binding[]; + primaryKeys: string[]; + calculatedAttributes: []; +} + +export interface SemanticEventSourceConfiguration { + active: boolean; + attributes: OntologyAttribute[]; + bindings: Binding[]; + primaryKeys: string[]; + timestampAttribute: string; + idAttribute: string; +} + +export interface Reference { + type: PigEntityReferenceType; + referenceKey: string; + namespace?: string; +} + +export interface ForeignKeyMapping { + sourceField: OntologyAttribute; + targetField: OntologyAttribute; +} + +export interface SemanticRelationshipConfiguration { + source: Reference; + target: Reference; + relationshipType: RelationshipType; + cardinality: Cardinality; + foreignKeyMappings: ForeignKeyMapping[]; +} + +export interface SemanticPerspectiveConfiguration { + active: boolean; + objects: Reference[]; + events: Reference[]; + relationships: Reference[]; + perspectiveType: PerspectiveType; + INSTANTIATE_ALL_EVENTS: boolean; +} diff --git a/src/commands/data-model-migration/module.ts b/src/commands/data-model-migration/module.ts new file mode 100644 index 00000000..e8a91007 --- /dev/null +++ b/src/commands/data-model-migration/module.ts @@ -0,0 +1,12 @@ +import { Configurator, IModule } from "../../core/command/module-handler"; +import { Context } from "../../core/command/cli-context"; +import { DataModelMigrationCommands } from "./data-model-migration.commands"; + +class Module extends IModule { + + public register(context: Context, configurator: Configurator): void { + new DataModelMigrationCommands().register(context, configurator); + } +} + +export = Module; diff --git a/src/commands/data-model-migration/service/data-model-converter.service.ts b/src/commands/data-model-migration/service/data-model-converter.service.ts new file mode 100644 index 00000000..5cae7716 --- /dev/null +++ b/src/commands/data-model-migration/service/data-model-converter.service.ts @@ -0,0 +1,347 @@ +import { SaveNodeTransport, NodeConfiguration } from "../../configuration-management/interfaces/node.interfaces"; +import { SEMANTIC_NODE_TYPES, SEMANTIC_SCHEMA_VERSION } from "../constants/semantic-node.constants"; +import { + ColumnType, + DataModelConfigurationTransport, + DataModelForeignKeyTransport, + DataModelTableTransport, + DataModelTransport, +} from "../interfaces/data-model-transport.interfaces"; +import { ConversionOptions, ConversionResult } from "../interfaces/conversion-result.interfaces"; +import { + AttributeDataType, + Binding, + ForeignKeyMapping, + OntologyAttribute, + Reference, + SemanticEventSourceConfiguration, + SemanticObjectConfiguration, + SemanticPerspectiveConfiguration, + SemanticRelationshipConfiguration, +} from "../interfaces/semantic-entity.interfaces"; + +interface TableContext { + table: DataModelTableTransport; + objectKey: string; + columnToAttributeId: Map; + attributeById: Map; +} + +/** Converts a Data Integration data model transport into Pacman staging node payloads. */ +export class DataModelConverterService { + + /** Converts tables, process configurations, and classic foreign keys into semantic entity nodes. */ + public convert(transport: DataModelTransport, options: ConversionOptions): ConversionResult { + const tableContexts = this.buildTableContexts(transport.tables ?? []); + const objects = tableContexts.map((ctx) => this.convertTable(ctx, options)); + const eventSources = (transport.processConfigurations ?? []) + .map((config) => this.convertProcessConfiguration(config, tableContexts, options)) + .filter((node): node is SaveNodeTransport => node !== null); + const relationships = (transport.foreignKeys ?? []) + .map((fk) => this.convertForeignKey(fk, tableContexts, options)) + .filter((node): node is SaveNodeTransport => node !== null); + const perspective = this.convertPerspective(transport, objects, eventSources, relationships, options); + + return { objects, eventSources, relationships, perspective }; + } + + /** Derives the lake binding schema from pool id unless an explicit schema is provided. */ + public static deriveBindingSchema(poolId: string, explicitSchema?: string): string { + if (explicitSchema) { + return explicitSchema; + } + return `datapipelines_${poolId.replace(/-/g, "_")}_draft`; + } + + private buildTableContexts(tables: DataModelTableTransport[]): TableContext[] { + return tables.map((table) => { + const objectKey = sanitizeKey(table.name); + const columnToAttributeId = new Map(); + const attributeById = new Map(); + const primaryKeys = table.primaryKeys ?? []; + + for (const column of table.columns ?? []) { + const attributeId = this.resolveAttributeId(column.name, primaryKeys); + columnToAttributeId.set(column.name, attributeId); + const attribute: OntologyAttribute = { + id: attributeId, + dataType: mapColumnType(column.type), + required: attributeId === "ID" || column.primaryKey === true, + }; + attributeById.set(attributeId, attribute); + } + + this.ensureIdAttribute(table, columnToAttributeId, attributeById); + + return { table, objectKey, columnToAttributeId, attributeById }; + }); + } + + private convertTable(ctx: TableContext, options: ConversionOptions): SaveNodeTransport { + const attributes = Array.from(ctx.attributeById.values()); + const primaryKeys = attributes.some((attribute) => attribute.id === "ID") ? ["ID"] : []; + const configuration: SemanticObjectConfiguration = { + active: true, + attributes, + primaryKeys, + calculatedAttributes: [], + bindings: [this.buildTableBinding(ctx, options)], + }; + + return buildSemanticNode( + options.packageKey, + ctx.objectKey, + ctx.table.name, + SEMANTIC_NODE_TYPES.OBJECT, + configuration + ); + } + + private convertProcessConfiguration( + config: DataModelConfigurationTransport, + tableContexts: TableContext[], + options: ConversionOptions + ): SaveNodeTransport | null { + const activityTable = tableContexts.find((ctx) => ctx.table.id === config.activityTableId); + if (!activityTable) { + return null; + } + + const columnNames = [ + config.caseIdColumn, + config.activityColumn, + config.timestampColumn, + config.endTimestampColumn, + config.sortingColumn, + config.costColumn, + config.userColumn, + ].filter((value): value is string => !!value); + + const attributeById = new Map(); + const columnToAttributeId = new Map(); + + for (const columnName of columnNames) { + const attributeId = columnName === config.caseIdColumn + ? "ID" + : sanitizeKey(columnName); + columnToAttributeId.set(columnName, attributeId); + const sourceColumn = activityTable.table.columns?.find((column) => column.name === columnName); + attributeById.set(attributeId, { + id: attributeId, + dataType: sourceColumn ? mapColumnType(sourceColumn.type) : "STRING", + required: attributeId === "ID" || columnName === config.timestampColumn, + }); + } + + if (!attributeById.has("ID")) { + attributeById.set("ID", { id: "ID", dataType: "STRING", required: true }); + columnToAttributeId.set(config.caseIdColumn, "ID"); + } + + const timestampAttribute = columnToAttributeId.get(config.timestampColumn); + if (!timestampAttribute) { + return null; + } + + const eventSourceKey = sanitizeKey(`${activityTable.objectKey}-events`); + const mappingColumns = columnNames.map((columnName) => ({ + sourceColumn: columnName, + targetColumn: columnToAttributeId.get(columnName) ?? sanitizeKey(columnName), + })); + + const configuration: SemanticEventSourceConfiguration = { + active: true, + attributes: Array.from(attributeById.values()), + primaryKeys: ["ID"], + timestampAttribute, + idAttribute: "ID", + bindings: [{ + name: `${eventSourceKey}-binding`, + namespace: options.namespace, + schema: options.bindingSchema, + table: activityTable.table.name, + mappingColumns, + }], + }; + + return buildSemanticNode( + options.packageKey, + eventSourceKey, + `${activityTable.table.name} Events`, + SEMANTIC_NODE_TYPES.EVENT_SOURCE, + configuration + ); + } + + private convertForeignKey( + foreignKey: DataModelForeignKeyTransport, + tableContexts: TableContext[], + options: ConversionOptions + ): SaveNodeTransport | null { + const source = tableContexts.find((ctx) => ctx.table.id === foreignKey.sourceTableId); + const target = tableContexts.find((ctx) => ctx.table.id === foreignKey.targetTableId); + if (!source || !target) { + return null; + } + + const foreignKeyMappings: ForeignKeyMapping[] = (foreignKey.columns ?? []).map((column) => ({ + sourceField: this.attributeForColumn(source, column.sourceColumnName), + targetField: this.attributeForColumn(target, column.targetColumnName), + })); + + const relationshipKey = sanitizeKey(`rel-${source.objectKey}-${target.objectKey}-${foreignKey.id}`); + const configuration: SemanticRelationshipConfiguration = { + source: objectReference(source.objectKey), + target: objectReference(target.objectKey), + relationshipType: "INSTANCE_TO_INSTANCE", + cardinality: "MANY_TO_ONE", + foreignKeyMappings, + }; + + return buildSemanticNode( + options.packageKey, + relationshipKey, + `${source.table.name} -> ${target.table.name}`, + SEMANTIC_NODE_TYPES.RELATIONSHIP, + configuration + ); + } + + private convertPerspective( + transport: DataModelTransport, + objects: SaveNodeTransport[], + eventSources: SaveNodeTransport[], + relationships: SaveNodeTransport[], + options: ConversionOptions + ): SaveNodeTransport { + const perspectiveKey = sanitizeKey(transport.name || transport.id); + const configuration: SemanticPerspectiveConfiguration = { + active: true, + objects: objects.map((object) => objectReference(object.key)), + events: eventSources.map((eventSource) => eventSourceReference(eventSource.key)), + relationships: relationships.map((relationship) => relationshipReference(relationship.key)), + perspectiveType: "CACHED", + INSTANTIATE_ALL_EVENTS: false, + }; + + return buildSemanticNode( + options.packageKey, + perspectiveKey, + transport.name, + SEMANTIC_NODE_TYPES.PERSPECTIVE, + configuration + ); + } + + private buildTableBinding(ctx: TableContext, options: ConversionOptions): Binding { + const mappingColumns = (ctx.table.columns ?? []).map((column) => ({ + sourceColumn: column.name, + targetColumn: ctx.columnToAttributeId.get(column.name) ?? sanitizeKey(column.name), + })); + + return { + name: `${ctx.objectKey}-binding`, + namespace: options.namespace, + schema: options.bindingSchema, + table: ctx.table.name, + mappingColumns, + }; + } + + private ensureIdAttribute( + table: DataModelTableTransport, + columnToAttributeId: Map, + attributeById: Map + ): void { + if (attributeById.has("ID")) { + return; + } + + const primaryKey = (table.primaryKeys ?? [])[0] ?? (table.columns ?? [])[0]?.name; + if (!primaryKey) { + attributeById.set("ID", { id: "ID", dataType: "STRING", required: true }); + return; + } + + columnToAttributeId.set(primaryKey, "ID"); + const sourceColumn = (table.columns ?? []).find((column) => column.name === primaryKey); + attributeById.set("ID", { + id: "ID", + dataType: sourceColumn ? mapColumnType(sourceColumn.type) : "STRING", + required: true, + }); + } + + private resolveAttributeId(columnName: string, primaryKeys: string[]): string { + if (primaryKeys.length === 1 && primaryKeys[0] === columnName) { + return "ID"; + } + return sanitizeKey(columnName); + } + + private attributeForColumn(ctx: TableContext, columnName: string): OntologyAttribute { + const attributeId = ctx.columnToAttributeId.get(columnName) ?? sanitizeKey(columnName); + return ctx.attributeById.get(attributeId) ?? { + id: attributeId, + dataType: "STRING", + }; + } +} + +function buildSemanticNode( + packageKey: string, + key: string, + name: string, + type: string, + configuration: NodeConfiguration +): SaveNodeTransport { + return { + key, + name, + type, + parentNodeKey: packageKey, + schemaVersion: SEMANTIC_SCHEMA_VERSION, + configuration, + }; +} + +function objectReference(referenceKey: string): Reference { + return { type: "OBJECT", referenceKey }; +} + +function eventSourceReference(referenceKey: string): Reference { + return { type: "EVENT_SOURCE", referenceKey }; +} + +function relationshipReference(referenceKey: string): Reference { + return { type: "RELATIONSHIP", referenceKey }; +} + +function mapColumnType(columnType: ColumnType): AttributeDataType { + switch (columnType) { + case "INTEGER": + return "INTEGER"; + case "FLOAT": + return "DOUBLE"; + case "DATE": + return "DATE"; + case "DATETIME": + return "TIMESTAMP"; + case "TIME": + return "STRING"; + case "BOOLEAN": + return "BOOLEAN"; + case "STRING": + default: + return "STRING"; + } +} + +function sanitizeKey(value: string): string { + return value + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, "_") + .replace(/^_+|_+$/g, "") + .replace(/_+/g, "_"); +} diff --git a/src/commands/data-model-migration/service/data-model-migration.service.ts b/src/commands/data-model-migration/service/data-model-migration.service.ts new file mode 100644 index 00000000..0b4fd3f5 --- /dev/null +++ b/src/commands/data-model-migration/service/data-model-migration.service.ts @@ -0,0 +1,106 @@ +import { v4 as uuidv4 } from "uuid"; +import { NodeApi } from "../../configuration-management/api/node-api"; +import { SaveNodeTransport } from "../../configuration-management/interfaces/node.interfaces"; +import { FileService, fileService } from "../../../core/utils/file-service"; +import { logger } from "../../../core/utils/logger"; +import { Context } from "../../../core/command/cli-context"; +import { DataModelApi } from "../api/data-model-api"; +import { DataModelTransport } from "../interfaces/data-model-transport.interfaces"; +import { ConversionResult } from "../interfaces/conversion-result.interfaces"; +import { DataModelConverterService } from "./data-model-converter.service"; + +export class DataModelMigrationService { + + private dataModelApi: DataModelApi; + private nodeApi: NodeApi; + private converter: DataModelConverterService; + + constructor(context: Context) { + this.dataModelApi = new DataModelApi(context); + this.nodeApi = new NodeApi(context); + this.converter = new DataModelConverterService(); + } + + /** Downloads a data model transport payload from cloud-data-integration. */ + public async exportDataModel(poolId: string, dataModelId: string, outputToJsonFile: boolean): Promise { + const transport = await this.dataModelApi.findOneTransport(poolId, dataModelId, true); + const payload = JSON.stringify(transport, null, 4); + + if (outputToJsonFile) { + const fileName = `${uuidv4()}_data_model_${dataModelId}.json`; + fileService.writeToFileWithGivenName(payload, fileName); + logger.info(FileService.fileDownloadedMessage + fileName); + return; + } + + logger.info("Exported Data Model:\n" + payload); + } + + /** Converts a data model transport and pushes semantic entity nodes into a target package. */ + public async pushSemanticModel(options: PushSemanticModelOptions): Promise { + const transport = await this.loadTransport(options); + const bindingSchema = DataModelConverterService.deriveBindingSchema(options.poolId, options.schema); + const conversion = this.converter.convert(transport, { + poolId: options.poolId, + bindingSchema, + packageKey: options.packageKey, + namespace: options.namespace, + }); + + if (options.dryRun) { + this.logDryRun(conversion, options.outputToJsonFile); + return; + } + + await this.pushConversion(options.packageKey, conversion); + logger.info( + `Successfully pushed semantic model to package '${options.packageKey}': ` + + `${conversion.objects.length} objects, ` + + `${conversion.eventSources.length} event sources, ` + + `${conversion.relationships.length} relationships, ` + + "1 perspective" + ); + } + + private async loadTransport(options: PushSemanticModelOptions): Promise { + if (options.fromFile) { + return JSON.parse(fileService.readFile(options.fromFile)) as DataModelTransport; + } + return this.dataModelApi.findOneTransport(options.poolId, options.dataModelId, true); + } + + private async pushConversion(packageKey: string, conversion: ConversionResult): Promise { + const nodes: SaveNodeTransport[] = [ + ...conversion.objects, + ...conversion.eventSources, + ...conversion.relationships, + conversion.perspective, + ]; + + for (const node of nodes) { + await this.nodeApi.createStagingNode(packageKey, node, false); + } + } + + private logDryRun(conversion: ConversionResult, outputToJsonFile?: boolean): void { + const payload = JSON.stringify(conversion, null, 4); + if (outputToJsonFile) { + const fileName = `${uuidv4()}_semantic_model_dry_run.json`; + fileService.writeToFileWithGivenName(payload, fileName); + logger.info(FileService.fileDownloadedMessage + fileName); + return; + } + logger.info("Dry run semantic model conversion:\n" + payload); + } +} + +export interface PushSemanticModelOptions { + poolId: string; + dataModelId: string; + packageKey: string; + schema?: string; + namespace?: string; + fromFile?: string; + dryRun?: boolean; + outputToJsonFile?: boolean; +} diff --git a/tests/commands/data-model-migration/data-model-converter.service.spec.ts b/tests/commands/data-model-migration/data-model-converter.service.spec.ts new file mode 100644 index 00000000..35c76aef --- /dev/null +++ b/tests/commands/data-model-migration/data-model-converter.service.spec.ts @@ -0,0 +1,180 @@ +import { DataModelTransport } from "../../../src/commands/data-model-migration/interfaces/data-model-transport.interfaces"; +import { DataModelConverterService } from "../../../src/commands/data-model-migration/service/data-model-converter.service"; + +const POOL_ID = "pool-123"; +const PACKAGE_KEY = "my-package"; +const SCHEMA = "custom_schema"; + +const sampleTransport = (): DataModelTransport => ({ + id: "dm-1", + name: "Order To Cash", + poolId: POOL_ID, + dataModelType: "CASE_CENTRIC", + tables: [ + { + id: "table-orders", + name: "ORDERS", + primaryKeys: ["ORDER_ID"], + columns: [ + { name: "ORDER_ID", type: "STRING", primaryKey: true }, + { name: "CUSTOMER_ID", type: "STRING" }, + { name: "AMOUNT", type: "FLOAT" }, + ], + }, + { + id: "table-customers", + name: "CUSTOMERS", + primaryKeys: ["CUSTOMER_ID"], + columns: [ + { name: "CUSTOMER_ID", type: "STRING", primaryKey: true }, + { name: "NAME", type: "STRING" }, + ], + }, + { + id: "table-events", + name: "EVENTS", + primaryKeys: ["CASE_ID"], + columns: [ + { name: "CASE_ID", type: "STRING", primaryKey: true }, + { name: "ACTIVITY", type: "STRING" }, + { name: "EVENT_TIME", type: "DATETIME" }, + ], + }, + ], + foreignKeys: [ + { + id: "fk-1", + sourceTableId: "table-orders", + targetTableId: "table-customers", + columns: [{ sourceColumnName: "CUSTOMER_ID", targetColumnName: "CUSTOMER_ID" }], + }, + ], + processConfigurations: [ + { + activityTableId: "table-events", + caseIdColumn: "CASE_ID", + activityColumn: "ACTIVITY", + timestampColumn: "EVENT_TIME", + defaultConfiguration: true, + }, + ], +}); + +const conversionOptions = () => ({ + poolId: POOL_ID, + bindingSchema: SCHEMA, + packageKey: PACKAGE_KEY, +}); + +describe("DataModelConverterService", () => { + const converter = new DataModelConverterService(); + + it("Should map tables to semantic object nodes with ID attribute and bindings", () => { + // Arrange + const transport = sampleTransport(); + + // Act + const result = converter.convert(transport, conversionOptions()); + + // Assert + const orders = result.objects.find((object) => object.key === "orders"); + expect(orders).toBeDefined(); + expect(orders?.type).toBe("SEMANTIC_OBJECT_TYPE"); + expect(orders?.parentNodeKey).toBe(PACKAGE_KEY); + expect(orders?.schemaVersion).toBe(1); + expect(orders?.configuration?.attributes).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: "ID", dataType: "STRING", required: true }), + expect.objectContaining({ id: "customer_id", dataType: "STRING" }), + expect.objectContaining({ id: "amount", dataType: "DOUBLE" }), + ]) + ); + expect(orders?.configuration?.bindings[0]).toEqual(expect.objectContaining({ + schema: SCHEMA, + table: "ORDERS", + mappingColumns: expect.arrayContaining([ + { sourceColumn: "ORDER_ID", targetColumn: "ID" }, + ]), + })); + }); + + it("Should map process configurations to semantic event source nodes", () => { + // Arrange + const transport = sampleTransport(); + + // Act + const result = converter.convert(transport, conversionOptions()); + + // Assert + expect(result.eventSources).toHaveLength(1); + expect(result.eventSources[0].key).toBe("events_events"); + expect(result.eventSources[0].type).toBe("SEMANTIC_EVENT_SOURCE_TYPE"); + expect(result.eventSources[0].configuration?.timestampAttribute).toBe("event_time"); + expect(result.eventSources[0].configuration?.idAttribute).toBe("ID"); + expect(result.eventSources[0].configuration?.bindings[0].table).toBe("EVENTS"); + }); + + it("Should map classic foreign keys to semantic relationship nodes without junction tables", () => { + // Arrange + const transport = sampleTransport(); + + // Act + const result = converter.convert(transport, conversionOptions()); + + // Assert + expect(result.relationships).toHaveLength(1); + expect(result.relationships[0].type).toBe("SEMANTIC_RELATIONSHIP_TYPE"); + expect(result.relationships[0].configuration?.source).toEqual({ type: "OBJECT", referenceKey: "orders" }); + expect(result.relationships[0].configuration?.target).toEqual({ type: "OBJECT", referenceKey: "customers" }); + expect(result.relationships[0].configuration?.cardinality).toBe("MANY_TO_ONE"); + expect(result.relationships[0].configuration?.foreignKeyMappings).toHaveLength(1); + expect(result.relationships[0].configuration?.junctionTable).toBeUndefined(); + }); + + it("Should build a perspective node referencing created objects, event sources, and relationships", () => { + // Arrange + const transport = sampleTransport(); + + // Act + const result = converter.convert(transport, conversionOptions()); + + // Assert + expect(result.perspective.key).toBe("order_to_cash"); + expect(result.perspective.type).toBe("SEMANTIC_PERSPECTIVE_TYPE"); + expect(result.perspective.configuration?.perspectiveType).toBe("CACHED"); + expect(result.perspective.configuration?.objects).toHaveLength(3); + expect(result.perspective.configuration?.events).toHaveLength(1); + expect(result.perspective.configuration?.relationships).toHaveLength(1); + }); + + it("Should derive binding schema from pool id when no explicit schema is provided", () => { + // Act + const schema = DataModelConverterService.deriveBindingSchema("pool-123"); + + // Assert + expect(schema).toBe("datapipelines_pool_123_draft"); + }); + + it("Should map TIME columns to STRING because semantic schemas have no TIME type", () => { + // Arrange + const transport: DataModelTransport = { + ...sampleTransport(), + tables: [{ + id: "table-times", + name: "TIMES", + primaryKeys: ["ID"], + columns: [{ name: "ID", type: "STRING", primaryKey: true }, { name: "START_TIME", type: "TIME" }], + }], + foreignKeys: [], + processConfigurations: [], + }; + + // Act + const result = converter.convert(transport, conversionOptions()); + + // Assert + expect(result.objects[0].configuration?.attributes).toEqual( + expect.arrayContaining([expect.objectContaining({ id: "start_time", dataType: "STRING" })]) + ); + }); +}); diff --git a/tests/commands/data-model-migration/data-model-migration.service.spec.ts b/tests/commands/data-model-migration/data-model-migration.service.spec.ts new file mode 100644 index 00000000..789b1ba0 --- /dev/null +++ b/tests/commands/data-model-migration/data-model-migration.service.spec.ts @@ -0,0 +1,90 @@ +import { mockAxiosGet, mockAxiosPost, mockedAxiosInstance } from "../../utls/http-requests-mock"; +import { testContext } from "../../utls/test-context"; +import { DataModelApi } from "../../../src/commands/data-model-migration/api/data-model-api"; +import { DataModelMigrationService } from "../../../src/commands/data-model-migration/service/data-model-migration.service"; +import { loggingTestTransport } from "../../jest.setup"; + +const POOL_ID = "pool-1"; +const DATA_MODEL_ID = "dm-1"; +const PACKAGE_KEY = "my-package"; +const TRANSPORT_URL = `https://myTeam.celonis.cloud/integration/api/pools/${POOL_ID}/data-models/${DATA_MODEL_ID}/transport?includeColumns=true`; +const NODE_CREATE_URL = `https://myTeam.celonis.cloud/pacman/api/core/staging/packages/${PACKAGE_KEY}/nodes`; + +const transportPayload = { + id: DATA_MODEL_ID, + name: "Demo Model", + tables: [{ + id: "t1", + name: "ORDERS", + primaryKeys: ["ORDER_ID"], + columns: [{ name: "ORDER_ID", type: "STRING", primaryKey: true }], + }], + foreignKeys: [], + processConfigurations: [], +}; + +describe("Data model migration APIs", () => { + it("Should call the integration transport endpoint with includeColumns=true", async () => { + // Arrange + mockAxiosGet(TRANSPORT_URL, transportPayload); + + // Act + const result = await new DataModelApi(testContext).findOneTransport(POOL_ID, DATA_MODEL_ID, true); + + // Assert + expect(mockedAxiosInstance.get).toHaveBeenCalledWith(TRANSPORT_URL, expect.anything()); + expect(result.id).toBe(DATA_MODEL_ID); + }); +}); + +describe("DataModelMigrationService push", () => { + beforeEach(() => { + mockAxiosGet(TRANSPORT_URL, transportPayload); + }); + + it("Should push converted entities in dependency order during a dry run without POST calls", async () => { + // Act + await new DataModelMigrationService(testContext).pushSemanticModel({ + poolId: POOL_ID, + dataModelId: DATA_MODEL_ID, + packageKey: PACKAGE_KEY, + dryRun: true, + }); + + // Assert + expect(mockedAxiosInstance.post).not.toHaveBeenCalled(); + expect(loggingTestTransport.logMessages[0].message).toContain("Dry run semantic model conversion"); + }); + + it("Should POST staging nodes to Pacman when dry run is disabled", async () => { + // Arrange + mockAxiosPost(NODE_CREATE_URL, { key: "orders" }); + mockAxiosPost(NODE_CREATE_URL, { key: "demo_model" }); + + // Act + await new DataModelMigrationService(testContext).pushSemanticModel({ + poolId: POOL_ID, + dataModelId: DATA_MODEL_ID, + packageKey: PACKAGE_KEY, + schema: "lake_schema", + }); + + // Assert + const postCalls = (mockedAxiosInstance.post as jest.Mock).mock.calls; + expect(postCalls).toHaveLength(2); + expect(postCalls[0][0]).toBe(NODE_CREATE_URL); + expect(JSON.parse(postCalls[0][1])).toEqual(expect.objectContaining({ + key: "orders", + type: "SEMANTIC_OBJECT_TYPE", + parentNodeKey: PACKAGE_KEY, + schemaVersion: 1, + })); + expect(postCalls[1][0]).toBe(NODE_CREATE_URL); + expect(JSON.parse(postCalls[1][1])).toEqual(expect.objectContaining({ + key: "demo_model", + type: "SEMANTIC_PERSPECTIVE_TYPE", + parentNodeKey: PACKAGE_KEY, + })); + expect(loggingTestTransport.logMessages[0].message).toContain("Successfully pushed semantic model"); + }); +}); diff --git a/tests/integration/commands/data-model-migration.spec.ts b/tests/integration/commands/data-model-migration.spec.ts new file mode 100644 index 00000000..2e83e834 --- /dev/null +++ b/tests/integration/commands/data-model-migration.spec.ts @@ -0,0 +1,52 @@ +import Module = require("../../../src/commands/data-model-migration/module"); +import { DataModelMigrationCommandService } from "../../../src/commands/data-model-migration/data-model-migration-command.service"; +import { runCli } from "../../utls/cli-runner"; + +jest.mock("../../../src/commands/data-model-migration/data-model-migration-command.service"); + +describe("data-model-migration command integration", () => { + let mockCommandService: jest.Mocked; + + beforeEach(() => { + mockCommandService = { + exportDataModel: jest.fn().mockResolvedValue(undefined), + pushSemanticModel: jest.fn().mockResolvedValue(undefined), + } as any; + (DataModelMigrationCommandService as jest.Mock).mockImplementation(() => mockCommandService); + }); + + it("Should wire export data-model to the command service", async () => { + // Act + await runCli(["export", "data-model", "--poolId", "pool-1", "--dataModelId", "dm-1"], [Module]); + + // Assert + expect(mockCommandService.exportDataModel).toHaveBeenCalledWith("pool-1", "dm-1", false); + }); + + it("Should wire push semantic-model to the command service", async () => { + // Act + await runCli([ + "push", + "semantic-model", + "--poolId", + "pool-1", + "--dataModelId", + "dm-1", + "--package", + "pkg-1", + "--dryRun", + ], [Module]); + + // Assert + expect(mockCommandService.pushSemanticModel).toHaveBeenCalledWith({ + poolId: "pool-1", + dataModelId: "dm-1", + packageKey: "pkg-1", + schema: undefined, + namespace: undefined, + fromFile: undefined, + dryRun: true, + outputToJsonFile: false, + }); + }); +});