From 89d90dd32bb90f449d66eb313e5d8d6893afd458 Mon Sep 17 00:00:00 2001 From: Philipp Hoch Date: Thu, 6 Aug 2026 13:21:47 +0200 Subject: [PATCH 1/3] Add data-model-migration commands for semantic entity push Introduce a new content-cli module that exports a Data Integration data model via the cloud-data-integration transport API, converts tables, process configurations, and classic foreign keys into pig semantic entities, and pushes them into a target pig-sl-ontology package. Includes converter unit tests, API/service tests, command wiring tests, and user guide documentation. Includes-AI-Code: true Co-authored-by: Cursor --- .github/CODEOWNERS | 2 + docs/command-graph.html | 10 +- docs/index.md | 1 + .../data-model-migration-commands.md | 70 ++++ .../api/data-model-api.ts | 23 ++ .../data-model-migration/api/ontology-api.ts | 65 ++++ .../data-model-migration-command.service.ts | 28 ++ .../data-model-migration.commands.ts | 55 +++ .../conversion-result.interfaces.ts | 20 ++ .../data-model-transport.interfaces.ts | 59 ++++ .../interfaces/ontology.interfaces.ts | 91 +++++ src/commands/data-model-migration/module.ts | 12 + .../service/data-model-converter.service.ts | 314 ++++++++++++++++++ .../service/data-model-migration.service.ts | 104 ++++++ src/core/profile/profile.service.ts | 1 + .../data-model-converter.service.spec.ts | 167 ++++++++++ .../data-model-migration.service.spec.ts | 90 +++++ .../commands/data-model-migration.spec.ts | 52 +++ 18 files changed, 1162 insertions(+), 2 deletions(-) create mode 100644 docs/user-guide/data-model-migration-commands.md create mode 100644 src/commands/data-model-migration/api/data-model-api.ts create mode 100644 src/commands/data-model-migration/api/ontology-api.ts create mode 100644 src/commands/data-model-migration/data-model-migration-command.service.ts create mode 100644 src/commands/data-model-migration/data-model-migration.commands.ts create mode 100644 src/commands/data-model-migration/interfaces/conversion-result.interfaces.ts create mode 100644 src/commands/data-model-migration/interfaces/data-model-transport.interfaces.ts create mode 100644 src/commands/data-model-migration/interfaces/ontology.interfaces.ts create mode 100644 src/commands/data-model-migration/module.ts create mode 100644 src/commands/data-model-migration/service/data-model-converter.service.ts create mode 100644 src/commands/data-model-migration/service/data-model-migration.service.ts create mode 100644 tests/commands/data-model-migration/data-model-converter.service.spec.ts create mode 100644 tests/commands/data-model-migration/data-model-migration.service.spec.ts create mode 100644 tests/integration/commands/data-model-migration.spec.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3f43cfe9..9003d021 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -17,6 +17,8 @@ /src/commands/analysis/ @celonis/process-analytics /src/commands/cpm4/ @celonis/cpm4 /src/commands/data-pipeline/ @Dusan-r @IvanGandacov @EktaCelonis @gorasoCelonis +/src/commands/data-model-migration/ @celonis/studio-platform +/tests/commands/data-model-migration/ @celonis/studio-platform /src/commands/studio/ @celonis/astro @celonis/studio-platform /tests/commands/studio/ @celonis/astro @celonis/studio-platform /package.json @celonis/studio-platform @aocelo @siavash-celonis diff --git a/docs/command-graph.html b/docs/command-graph.html index ed95d759..2a0d2c5b 100644 --- a/docs/command-graph.html +++ b/docs/command-graph.html @@ -211,6 +211,9 @@ { id: "export_data_pool", label: "data-pool", group: "command", path: "export data-pool", description: "Command to export a data pool", options: ["-p, --profile ", "--id ", "--outputToJsonFile", "-h, --help"] }, + { id: "export_data_model", label: "data-model", group: "command", path: "export data-model", + description: "Export a data model with tables, foreign keys, and process configurations", + options: ["-p, --profile ", "--poolId ", "--dataModelId ", "--outputToJsonFile", "-h, --help"] }, // import { id: "import_action_flows", label: "action-flows", group: "command", path: "import action-flows", @@ -247,6 +250,9 @@ description: "Command to push a data pool", options: ["-p, --profile ", "-f, --file ", "-h, --help"] }, { id: "push_data_pools", label: "data-pools", group: "command", path: "push data-pools", description: "Command to push data pools", options: ["-p, --profile ", "-h, --help"] }, + { id: "push_semantic_model", label: "semantic-model", group: "command", path: "push semantic-model", + description: "Convert a data model into semantic entities and push them into a pig package", + options: ["-p, --profile ", "--poolId ", "--dataModelId ", "--package ", "--schema ", "--namespace ", "-f, --fromFile ", "--dryRun", "--outputToJsonFile", "-h, --help"] }, { id: "push_asset", label: "asset", group: "command", path: "push asset", description: "Command to push an asset to Studio", options: ["-p, --profile ", "-f, --file ", "--package ", "-h, --help"] }, { id: "push_assets", label: "assets", group: "command", path: "push assets", @@ -479,7 +485,7 @@ ["area_analyze","analyze_action_flows"], - ["area_export","export_action_flows"],["area_export","export_data_pool"], + ["area_export","export_action_flows"],["area_export","export_data_pool"],["area_export","export_data_model"], ["area_import","import_action_flows"],["area_import","import_data_pools"], @@ -487,7 +493,7 @@ ["area_pull","pull_asset"],["area_pull","pull_package"],["area_pull","pull_view_bookmarks"], ["area_push","push_skill"],["area_push","push_bookmarks"],["area_push","push_ctp"], - ["area_push","push_data_pool"],["area_push","push_data_pools"],["area_push","push_asset"], + ["area_push","push_data_pool"],["area_push","push_data_pools"],["area_push","push_semantic_model"],["area_push","push_asset"], ["area_push","push_assets"],["area_push","push_package"],["area_push","push_packages"], ["area_push","push_widget"],["area_push","push_view_bookmarks"], diff --git a/docs/index.md b/docs/index.md index 1e47cd50..abac28ec 100644 --- a/docs/index.md +++ b/docs/index.md @@ -53,6 +53,7 @@ content-cli pull package -h - [Config Commands](user-guide/config-commands.md) -- Batch export/import, variables, nodes, diffs, and dependencies - [Deployment Commands](user-guide/deployment-commands.md) -- Create, history, active deployments, deployables, and targets - [Data Pool Commands](user-guide/data-pool-commands.md) -- Export/import data pools and connection management + - [Data Model Migration Commands](user-guide/data-model-migration-commands.md) -- Export data models and push semantic entities to pig packages - [Action Flow Commands](user-guide/action-flow-commands.md) -- Analyze and export action flows - **Development** - [Architecture](internal-architecture.md) -- Internal architecture and inner workings diff --git a/docs/user-guide/data-model-migration-commands.md b/docs/user-guide/data-model-migration-commands.md new file mode 100644 index 00000000..03de3dc1 --- /dev/null +++ b/docs/user-guide/data-model-migration-commands.md @@ -0,0 +1,70 @@ +# Data Model Migration Commands + +These commands export a Data Integration data model from cloud-data-integration and convert it into semantic entities in a target pig package via pig-sl-ontology. + +Supported mappings: + +| Data Integration | Semantic entity | +|---|---| +| Table | Object (with data binding) | +| Process configuration (event log) | Event source (with data binding) | +| Classic foreign key | Relationship | +| Data model | Perspective | + +Object-centric **object links** (`signal-links` in cloud-data-integration) are **not** supported. The converter only reads classic `foreignKeys[]` from the `/transport` export. + +## Export Data Model + +Downloads the full data model transport (tables, columns, foreign keys, process configurations): + +``` +content-cli export data-model --poolId --dataModelId --profile [--outputToJsonFile] +``` + +Example: + +``` +content-cli export data-model --poolId 80a1389d-50c5-4976-ad6e-fb5b7a2b5517 --dataModelId 1b9b368b-e0df-4e74-99e8-59e2febe9687 --profile local --outputToJsonFile +``` + +## Push Semantic Model + +Converts the data model and pushes semantic entities into a target pig package: + +``` +content-cli push semantic-model \ + --poolId \ + --dataModelId \ + --package \ + --profile \ + [--schema ] \ + [--namespace ] \ + [--fromFile ] \ + [--dryRun] \ + [--outputToJsonFile] +``` + +Options: + +- `--schema`: Physical lake schema used in data bindings. When omitted, the CLI derives `datapipelines__draft` (hyphens in the pool id become underscores). +- `--fromFile`: Skip download and convert a previously exported transport JSON file. +- `--dryRun`: Convert only; print or write the ontology payloads without calling pig-sl-ontology. +- `--namespace`: Optional namespace for created semantic entities (defaults to ontology `"local"` when omitted). + +Push order: objects → event sources → relationships → perspective. + +Example dry run: + +``` +content-cli push semantic-model \ + --poolId 80a1389d-50c5-4976-ad6e-fb5b7a2b5517 \ + --dataModelId 1b9b368b-e0df-4e74-99e8-59e2febe9687 \ + --package my-context-model \ + --profile local \ + --dryRun +``` + +## Authentication + +- **Download** uses the existing `integration.data-pools` OAuth scope (same as other data pool commands). +- **Push** calls `/pig-sl-ontology/api/ontology/packages/{packageKey}/semantic-*` with the profile bearer token or API key. Ensure the profile has edit access to the target package. diff --git a/src/commands/data-model-migration/api/data-model-api.ts b/src/commands/data-model-migration/api/data-model-api.ts new file mode 100644 index 00000000..2ac2d5f8 --- /dev/null +++ b/src/commands/data-model-migration/api/data-model-api.ts @@ -0,0 +1,23 @@ +import { Context } from "../../../core/command/cli-context"; +import { FatalError } from "../../../core/utils/logger"; +import { HttpClient } from "../../../core/http/http-client"; +import { DataModelTransport } from "../interfaces/data-model-transport.interfaces"; + +export class DataModelApi { + + private httpClient: () => HttpClient; + + constructor(context: Context) { + this.httpClient = () => context.httpClient; + } + + /** Fetches the full data model transport including columns from cloud-data-integration. */ + public async findOneTransport(poolId: string, dataModelId: string, includeColumns = true): Promise { + const query = includeColumns ? "?includeColumns=true" : ""; + return this.httpClient() + .get(`/integration/api/pools/${poolId}/data-models/${dataModelId}/transport${query}`) + .catch((error) => { + throw new FatalError(`Data model export failed for pool ${poolId}, data model ${dataModelId}: ${error}`); + }); + } +} diff --git a/src/commands/data-model-migration/api/ontology-api.ts b/src/commands/data-model-migration/api/ontology-api.ts new file mode 100644 index 00000000..f5e87536 --- /dev/null +++ b/src/commands/data-model-migration/api/ontology-api.ts @@ -0,0 +1,65 @@ +import { Context } from "../../../core/command/cli-context"; +import { FatalError } from "../../../core/utils/logger"; +import { HttpClient } from "../../../core/http/http-client"; +import { + OntologyNodeRequest, + OntologyNodeResponse, + SemanticEventSourceContent, + SemanticObjectContent, + SemanticPerspectiveContent, + SemanticRelationshipContent, +} from "../interfaces/ontology.interfaces"; + +const ONTOLOGY_BASE = "/pig-sl-ontology/api/ontology/packages"; + +export class OntologyApi { + + private httpClient: () => HttpClient; + + constructor(context: Context) { + this.httpClient = () => context.httpClient; + } + + /** Creates a semantic object in the target package. */ + public async createObject( + packageKey: string, + request: OntologyNodeRequest + ): Promise> { + return this.post(`${ONTOLOGY_BASE}/${packageKey}/semantic-objects`, request); + } + + /** Creates a semantic event source in the target package. */ + public async createEventSource( + packageKey: string, + request: OntologyNodeRequest + ): Promise> { + return this.post(`${ONTOLOGY_BASE}/${packageKey}/semantic-event-sources`, request); + } + + /** Creates a semantic relationship in the target package. */ + public async createRelationship( + packageKey: string, + request: OntologyNodeRequest + ): Promise> { + return this.post(`${ONTOLOGY_BASE}/${packageKey}/semantic-relationships`, request); + } + + /** Creates a semantic perspective in the target package. */ + public async createPerspective( + packageKey: string, + request: OntologyNodeRequest + ): Promise> { + return this.post(`${ONTOLOGY_BASE}/${packageKey}/semantic-perspectives`, request); + } + + private post( + url: string, + request: OntologyNodeRequest + ): Promise { + return this.httpClient() + .post(url, request) + .catch((error) => { + throw new FatalError(`Problem creating semantic entity '${request.key}': ${error}`); + }); + } +} diff --git a/src/commands/data-model-migration/data-model-migration-command.service.ts b/src/commands/data-model-migration/data-model-migration-command.service.ts new file mode 100644 index 00000000..e4400d67 --- /dev/null +++ b/src/commands/data-model-migration/data-model-migration-command.service.ts @@ -0,0 +1,28 @@ +import { Context } from "../../core/command/cli-context"; +import { DataModelMigrationService } from "./service/data-model-migration.service"; + +export class DataModelMigrationCommandService { + + private migrationService: DataModelMigrationService; + + constructor(context: Context) { + this.migrationService = new DataModelMigrationService(context); + } + + public async exportDataModel(poolId: string, dataModelId: string, outputToJsonFile: boolean): Promise { + await this.migrationService.exportDataModel(poolId, dataModelId, outputToJsonFile); + } + + public async pushSemanticModel(options: { + poolId: string; + dataModelId: string; + packageKey: string; + schema?: string; + namespace?: string; + fromFile?: string; + dryRun?: boolean; + outputToJsonFile?: boolean; + }): Promise { + await this.migrationService.pushSemanticModel(options); + } +} diff --git a/src/commands/data-model-migration/data-model-migration.commands.ts b/src/commands/data-model-migration/data-model-migration.commands.ts new file mode 100644 index 00000000..bfdbab58 --- /dev/null +++ b/src/commands/data-model-migration/data-model-migration.commands.ts @@ -0,0 +1,55 @@ +import { Context } from "../../core/command/cli-context"; +import { Configurator } from "../../core/command/module-handler"; +import { Command, OptionValues } from "commander"; +import { DataModelMigrationCommandService } from "./data-model-migration-command.service"; + +export class DataModelMigrationCommands { + + public register(_context: Context, configurator: Configurator): void { + configurator.command("export") + .command("data-model") + .description("Export a data model with tables, foreign keys, and process configurations") + .requiredOption("--poolId ", "ID of the data pool") + .requiredOption("--dataModelId ", "ID of the data model") + .option("--outputToJsonFile", "Write the exported data model to a JSON file") + .action(this.exportDataModel); + + configurator.command("push") + .command("semantic-model") + .description("Convert a data model into semantic entities and push them into a pig package") + .requiredOption("--package ", "Target pig package key") + .option("--poolId ", "ID of the data pool (required unless --fromFile is set)") + .option("--dataModelId ", "ID of the data model (required unless --fromFile is set)") + .option("--schema ", "Physical lake schema for data bindings (overrides pool-derived default)") + .option("--namespace ", "Namespace for created semantic entities") + .option("-f, --fromFile ", "Use a previously exported data model transport JSON file") + .option("--dryRun", "Convert only; print or write payloads without pushing to ontology") + .option("--outputToJsonFile", "With --dryRun, write conversion output to a JSON file") + .action(this.pushSemanticModel); + } + + private async exportDataModel(context: Context, _command: Command, options: OptionValues): Promise { + await new DataModelMigrationCommandService(context).exportDataModel( + options.poolId, + options.dataModelId, + !!options.outputToJsonFile + ); + } + + private async pushSemanticModel(context: Context, _command: Command, options: OptionValues): Promise { + if (!options.fromFile && (!options.poolId || !options.dataModelId)) { + throw new Error("Either --fromFile or both --poolId and --dataModelId are required"); + } + + await new DataModelMigrationCommandService(context).pushSemanticModel({ + poolId: options.poolId, + dataModelId: options.dataModelId, + packageKey: options.package, + schema: options.schema, + namespace: options.namespace, + fromFile: options.fromFile, + dryRun: !!options.dryRun, + outputToJsonFile: !!options.outputToJsonFile, + }); + } +} diff --git a/src/commands/data-model-migration/interfaces/conversion-result.interfaces.ts b/src/commands/data-model-migration/interfaces/conversion-result.interfaces.ts new file mode 100644 index 00000000..0130aba3 --- /dev/null +++ b/src/commands/data-model-migration/interfaces/conversion-result.interfaces.ts @@ -0,0 +1,20 @@ +import { + OntologyNodeRequest, + SemanticEventSourceContent, + SemanticObjectContent, + SemanticPerspectiveContent, + SemanticRelationshipContent, +} from "./ontology.interfaces"; + +export interface ConversionResult { + objects: OntologyNodeRequest[]; + eventSources: OntologyNodeRequest[]; + relationships: OntologyNodeRequest[]; + perspective: OntologyNodeRequest; +} + +export interface ConversionOptions { + poolId: string; + bindingSchema: string; + namespace?: string; +} diff --git a/src/commands/data-model-migration/interfaces/data-model-transport.interfaces.ts b/src/commands/data-model-migration/interfaces/data-model-transport.interfaces.ts new file mode 100644 index 00000000..87c65958 --- /dev/null +++ b/src/commands/data-model-migration/interfaces/data-model-transport.interfaces.ts @@ -0,0 +1,59 @@ +export type ColumnType = "INTEGER" | "DATE" | "TIME" | "DATETIME" | "FLOAT" | "BOOLEAN" | "STRING"; + +export type DataModelType = "CASE_CENTRIC" | "OBJECT_CENTRIC"; + +export interface DataModelColumnTransport { + name: string; + type: ColumnType; + primaryKey?: boolean; +} + +export interface DataModelTableTransport { + id: string; + name: string; + alias?: string; + aliasOrName?: string; + dataModelId?: string; + dataSourceId?: string; + primaryKeys?: string[]; + columns?: DataModelColumnTransport[]; +} + +export interface DataModelForeignKeyColumnTransport { + id?: string; + sourceColumnName: string; + targetColumnName: string; +} + +export interface DataModelForeignKeyTransport { + id: string; + dataModelId?: string; + sourceTableId: string; + targetTableId: string; + columns: DataModelForeignKeyColumnTransport[]; +} + +export interface DataModelConfigurationTransport { + id?: string; + dataModelId?: string; + activityTableId: string; + caseTableId?: string; + caseIdColumn: string; + activityColumn: string; + timestampColumn: string; + endTimestampColumn?: string; + sortingColumn?: string; + costColumn?: string; + userColumn?: string; + defaultConfiguration?: boolean; +} + +export interface DataModelTransport { + id: string; + name: string; + poolId?: string; + dataModelType?: DataModelType; + tables: DataModelTableTransport[]; + foreignKeys?: DataModelForeignKeyTransport[]; + processConfigurations?: DataModelConfigurationTransport[]; +} diff --git a/src/commands/data-model-migration/interfaces/ontology.interfaces.ts b/src/commands/data-model-migration/interfaces/ontology.interfaces.ts new file mode 100644 index 00000000..7a6b0a8b --- /dev/null +++ b/src/commands/data-model-migration/interfaces/ontology.interfaces.ts @@ -0,0 +1,91 @@ +export type AttributeDataType = + | "STRING" + | "INTEGER" + | "LONG" + | "DOUBLE" + | "BOOLEAN" + | "TIMESTAMP" + | "DATE"; + +export type PigEntityReferenceType = "OBJECT" | "EVENT_SOURCE" | "RELATIONSHIP"; + +export type RelationshipType = "TYPE_TO_TYPE" | "INSTANCE_TO_INSTANCE"; + +export type Cardinality = "ONE_TO_ONE" | "ONE_TO_MANY" | "MANY_TO_ONE" | "MANY_TO_MANY"; + +export type PerspectiveType = "LIVE" | "CACHED"; + +export interface OntologyAttribute { + id: string; + dataType: AttributeDataType; + required?: boolean; + description?: string; +} + +export interface MappingColumn { + sourceColumn: string; + targetColumn: string; +} + +export interface Binding { + name: string; + namespace?: string; + schema: string; + table: string; + mappingColumns: MappingColumn[]; +} + +export interface SemanticObjectContent { + attributes: OntologyAttribute[]; + bindings: Binding[]; + primaryKeys?: string[]; +} + +export interface SemanticEventSourceContent { + attributes: OntologyAttribute[]; + bindings: Binding[]; + timestampAttribute: string; + idAttribute: string; + primaryKeys?: string[]; +} + +export interface Reference { + type: PigEntityReferenceType; + referenceKey: string; + namespace?: string; +} + +export interface ForeignKeyMapping { + sourceField: OntologyAttribute; + targetField: OntologyAttribute; +} + +export interface SemanticRelationshipContent { + source: Reference; + target: Reference; + relationshipType?: RelationshipType; + cardinality?: Cardinality; + foreignKeyMappings: ForeignKeyMapping[]; +} + +export interface SemanticPerspectiveContent { + objects: Reference[]; + events: Reference[]; + relationships: Reference[]; + perspectiveType?: PerspectiveType; + INSTANTIATE_ALL_EVENTS?: boolean; +} + +export interface OntologyNodeRequest { + key: string; + name: string; + namespace?: string; + content: T; +} + +export interface OntologyNodeResponse { + key: string; + name: string; + packageNodeKey?: string; + content?: T; +} diff --git a/src/commands/data-model-migration/module.ts b/src/commands/data-model-migration/module.ts new file mode 100644 index 00000000..e8a91007 --- /dev/null +++ b/src/commands/data-model-migration/module.ts @@ -0,0 +1,12 @@ +import { Configurator, IModule } from "../../core/command/module-handler"; +import { Context } from "../../core/command/cli-context"; +import { DataModelMigrationCommands } from "./data-model-migration.commands"; + +class Module extends IModule { + + public register(context: Context, configurator: Configurator): void { + new DataModelMigrationCommands().register(context, configurator); + } +} + +export = Module; diff --git a/src/commands/data-model-migration/service/data-model-converter.service.ts b/src/commands/data-model-migration/service/data-model-converter.service.ts new file mode 100644 index 00000000..6c65ca1c --- /dev/null +++ b/src/commands/data-model-migration/service/data-model-converter.service.ts @@ -0,0 +1,314 @@ +import { + ColumnType, + DataModelConfigurationTransport, + DataModelForeignKeyTransport, + DataModelTableTransport, + DataModelTransport, +} from "../interfaces/data-model-transport.interfaces"; +import { ConversionOptions, ConversionResult } from "../interfaces/conversion-result.interfaces"; +import { + AttributeDataType, + Binding, + ForeignKeyMapping, + OntologyAttribute, + OntologyNodeRequest, + Reference, + SemanticEventSourceContent, + SemanticObjectContent, + SemanticPerspectiveContent, + SemanticRelationshipContent, +} from "../interfaces/ontology.interfaces"; + +interface TableContext { + table: DataModelTableTransport; + objectKey: string; + columnToAttributeId: Map; + attributeById: Map; +} + +/** Converts a Data Integration data model transport into pig semantic entity requests. */ +export class DataModelConverterService { + + /** Converts tables, process configurations, and classic foreign keys into semantic entities. */ + public convert(transport: DataModelTransport, options: ConversionOptions): ConversionResult { + const tableContexts = this.buildTableContexts(transport.tables ?? []); + const objects = tableContexts.map((ctx) => this.convertTable(ctx, options)); + const eventSources = (transport.processConfigurations ?? []) + .map((config) => this.convertProcessConfiguration(config, tableContexts, options)) + .filter((request): request is OntologyNodeRequest => request !== null); + const relationships = (transport.foreignKeys ?? []) + .map((fk) => this.convertForeignKey(fk, tableContexts)) + .filter((request): request is OntologyNodeRequest => request !== null); + const perspective = this.convertPerspective(transport, objects, eventSources, relationships); + + return { objects, eventSources, relationships, perspective }; + } + + /** Derives the lake binding schema from pool id unless an explicit schema is provided. */ + public static deriveBindingSchema(poolId: string, explicitSchema?: string): string { + if (explicitSchema) { + return explicitSchema; + } + return `datapipelines_${poolId.replace(/-/g, "_")}_draft`; + } + + private buildTableContexts(tables: DataModelTableTransport[]): TableContext[] { + return tables.map((table) => { + const objectKey = sanitizeKey(table.name); + const columnToAttributeId = new Map(); + const attributeById = new Map(); + const primaryKeys = table.primaryKeys ?? []; + + for (const column of table.columns ?? []) { + const attributeId = this.resolveAttributeId(column.name, primaryKeys); + columnToAttributeId.set(column.name, attributeId); + const attribute: OntologyAttribute = { + id: attributeId, + dataType: mapColumnType(column.type), + required: attributeId === "ID" || column.primaryKey === true, + }; + attributeById.set(attributeId, attribute); + } + + this.ensureIdAttribute(table, columnToAttributeId, attributeById); + + return { table, objectKey, columnToAttributeId, attributeById }; + }); + } + + private convertTable( + ctx: TableContext, + options: ConversionOptions + ): OntologyNodeRequest { + const attributes = Array.from(ctx.attributeById.values()); + const primaryKeys = attributes.some((attribute) => attribute.id === "ID") ? ["ID"] : []; + + return { + key: ctx.objectKey, + name: ctx.table.name, + namespace: options.namespace, + content: { + attributes, + primaryKeys, + bindings: [this.buildTableBinding(ctx, options.bindingSchema)], + }, + }; + } + + private convertProcessConfiguration( + config: DataModelConfigurationTransport, + tableContexts: TableContext[], + options: ConversionOptions + ): OntologyNodeRequest | null { + const activityTable = tableContexts.find((ctx) => ctx.table.id === config.activityTableId); + if (!activityTable) { + return null; + } + + const columnNames = [ + config.caseIdColumn, + config.activityColumn, + config.timestampColumn, + config.endTimestampColumn, + config.sortingColumn, + config.costColumn, + config.userColumn, + ].filter((value): value is string => !!value); + + const attributeById = new Map(); + const columnToAttributeId = new Map(); + + for (const columnName of columnNames) { + const attributeId = columnName === config.caseIdColumn + ? "ID" + : sanitizeKey(columnName); + columnToAttributeId.set(columnName, attributeId); + const sourceColumn = activityTable.table.columns?.find((column) => column.name === columnName); + attributeById.set(attributeId, { + id: attributeId, + dataType: sourceColumn ? mapColumnType(sourceColumn.type) : "STRING", + required: attributeId === "ID" || columnName === config.timestampColumn, + }); + } + + if (!attributeById.has("ID")) { + attributeById.set("ID", { id: "ID", dataType: "STRING", required: true }); + columnToAttributeId.set(config.caseIdColumn, "ID"); + } + + const timestampAttribute = columnToAttributeId.get(config.timestampColumn); + if (!timestampAttribute) { + return null; + } + + const eventSourceKey = sanitizeKey(`${activityTable.objectKey}-events`); + const mappingColumns = columnNames.map((columnName) => ({ + sourceColumn: columnName, + targetColumn: columnToAttributeId.get(columnName) ?? sanitizeKey(columnName), + })); + + return { + key: eventSourceKey, + name: `${activityTable.table.name} Events`, + namespace: options.namespace, + content: { + attributes: Array.from(attributeById.values()), + primaryKeys: ["ID"], + timestampAttribute, + idAttribute: "ID", + bindings: [{ + name: `${eventSourceKey}-binding`, + namespace: options.namespace, + schema: options.bindingSchema, + table: activityTable.table.name, + mappingColumns, + }], + }, + }; + } + + private convertForeignKey( + foreignKey: DataModelForeignKeyTransport, + tableContexts: TableContext[] + ): OntologyNodeRequest | null { + const source = tableContexts.find((ctx) => ctx.table.id === foreignKey.sourceTableId); + const target = tableContexts.find((ctx) => ctx.table.id === foreignKey.targetTableId); + if (!source || !target) { + return null; + } + + const foreignKeyMappings: ForeignKeyMapping[] = (foreignKey.columns ?? []).map((column) => ({ + sourceField: this.attributeForColumn(source, column.sourceColumnName), + targetField: this.attributeForColumn(target, column.targetColumnName), + })); + + const relationshipKey = sanitizeKey(`rel-${source.objectKey}-${target.objectKey}-${foreignKey.id}`); + + return { + key: relationshipKey, + name: `${source.table.name} -> ${target.table.name}`, + content: { + source: objectReference(source.objectKey), + target: objectReference(target.objectKey), + relationshipType: "INSTANCE_TO_INSTANCE", + cardinality: "MANY_TO_ONE", + foreignKeyMappings, + }, + }; + } + + private convertPerspective( + transport: DataModelTransport, + objects: OntologyNodeRequest[], + eventSources: OntologyNodeRequest[], + relationships: OntologyNodeRequest[] + ): OntologyNodeRequest { + const perspectiveKey = sanitizeKey(transport.name || transport.id); + + return { + key: perspectiveKey, + name: transport.name, + content: { + objects: objects.map((object) => objectReference(object.key)), + events: eventSources.map((eventSource) => eventSourceReference(eventSource.key)), + relationships: relationships.map((relationship) => relationshipReference(relationship.key)), + perspectiveType: "CACHED", + INSTANTIATE_ALL_EVENTS: false, + }, + }; + } + + private buildTableBinding(ctx: TableContext, schema: string): Binding { + const mappingColumns = (ctx.table.columns ?? []).map((column) => ({ + sourceColumn: column.name, + targetColumn: ctx.columnToAttributeId.get(column.name) ?? sanitizeKey(column.name), + })); + + return { + name: `${ctx.objectKey}-binding`, + schema, + table: ctx.table.name, + mappingColumns, + }; + } + + private ensureIdAttribute( + table: DataModelTableTransport, + columnToAttributeId: Map, + attributeById: Map + ): void { + if (attributeById.has("ID")) { + return; + } + + const primaryKey = (table.primaryKeys ?? [])[0] ?? (table.columns ?? [])[0]?.name; + if (!primaryKey) { + attributeById.set("ID", { id: "ID", dataType: "STRING", required: true }); + return; + } + + columnToAttributeId.set(primaryKey, "ID"); + const sourceColumn = (table.columns ?? []).find((column) => column.name === primaryKey); + attributeById.set("ID", { + id: "ID", + dataType: sourceColumn ? mapColumnType(sourceColumn.type) : "STRING", + required: true, + }); + } + + private resolveAttributeId(columnName: string, primaryKeys: string[]): string { + if (primaryKeys.length === 1 && primaryKeys[0] === columnName) { + return "ID"; + } + return sanitizeKey(columnName); + } + + private attributeForColumn(ctx: TableContext, columnName: string): OntologyAttribute { + const attributeId = ctx.columnToAttributeId.get(columnName) ?? sanitizeKey(columnName); + return ctx.attributeById.get(attributeId) ?? { + id: attributeId, + dataType: "STRING", + }; + } +} + +function objectReference(referenceKey: string): Reference { + return { type: "OBJECT", referenceKey }; +} + +function eventSourceReference(referenceKey: string): Reference { + return { type: "EVENT_SOURCE", referenceKey }; +} + +function relationshipReference(referenceKey: string): Reference { + return { type: "RELATIONSHIP", referenceKey }; +} + +function mapColumnType(columnType: ColumnType): AttributeDataType { + switch (columnType) { + case "INTEGER": + return "INTEGER"; + case "FLOAT": + return "DOUBLE"; + case "DATE": + return "DATE"; + case "DATETIME": + return "TIMESTAMP"; + case "TIME": + return "STRING"; + case "BOOLEAN": + return "BOOLEAN"; + case "STRING": + default: + return "STRING"; + } +} + +function sanitizeKey(value: string): string { + return value + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, "_") + .replace(/^_+|_+$/g, "") + .replace(/_+/g, "_"); +} diff --git a/src/commands/data-model-migration/service/data-model-migration.service.ts b/src/commands/data-model-migration/service/data-model-migration.service.ts new file mode 100644 index 00000000..bab0c0fe --- /dev/null +++ b/src/commands/data-model-migration/service/data-model-migration.service.ts @@ -0,0 +1,104 @@ +import { v4 as uuidv4 } from "uuid"; +import { FileService, fileService } from "../../../core/utils/file-service"; +import { logger } from "../../../core/utils/logger"; +import { Context } from "../../../core/command/cli-context"; +import { DataModelApi } from "../api/data-model-api"; +import { OntologyApi } from "../api/ontology-api"; +import { DataModelTransport } from "../interfaces/data-model-transport.interfaces"; +import { ConversionResult } from "../interfaces/conversion-result.interfaces"; +import { DataModelConverterService } from "./data-model-converter.service"; + +export class DataModelMigrationService { + + private dataModelApi: DataModelApi; + private ontologyApi: OntologyApi; + private converter: DataModelConverterService; + + constructor(context: Context) { + this.dataModelApi = new DataModelApi(context); + this.ontologyApi = new OntologyApi(context); + this.converter = new DataModelConverterService(); + } + + /** Downloads a data model transport payload from cloud-data-integration. */ + public async exportDataModel(poolId: string, dataModelId: string, outputToJsonFile: boolean): Promise { + const transport = await this.dataModelApi.findOneTransport(poolId, dataModelId, true); + const payload = JSON.stringify(transport, null, 4); + + if (outputToJsonFile) { + const fileName = `${uuidv4()}_data_model_${dataModelId}.json`; + fileService.writeToFileWithGivenName(payload, fileName); + logger.info(FileService.fileDownloadedMessage + fileName); + return; + } + + logger.info("Exported Data Model:\n" + payload); + } + + /** Converts a data model transport and pushes semantic entities into a target package. */ + public async pushSemanticModel(options: PushSemanticModelOptions): Promise { + const transport = await this.loadTransport(options); + const bindingSchema = DataModelConverterService.deriveBindingSchema(options.poolId, options.schema); + const conversion = this.converter.convert(transport, { + poolId: options.poolId, + bindingSchema, + namespace: options.namespace, + }); + + if (options.dryRun) { + this.logDryRun(conversion, options.outputToJsonFile); + return; + } + + await this.pushConversion(options.packageKey, conversion); + logger.info( + `Successfully pushed semantic model to package '${options.packageKey}': ` + + `${conversion.objects.length} objects, ` + + `${conversion.eventSources.length} event sources, ` + + `${conversion.relationships.length} relationships, ` + + "1 perspective" + ); + } + + private async loadTransport(options: PushSemanticModelOptions): Promise { + if (options.fromFile) { + return JSON.parse(fileService.readFile(options.fromFile)) as DataModelTransport; + } + return this.dataModelApi.findOneTransport(options.poolId, options.dataModelId, true); + } + + private async pushConversion(packageKey: string, conversion: ConversionResult): Promise { + for (const object of conversion.objects) { + await this.ontologyApi.createObject(packageKey, object); + } + for (const eventSource of conversion.eventSources) { + await this.ontologyApi.createEventSource(packageKey, eventSource); + } + for (const relationship of conversion.relationships) { + await this.ontologyApi.createRelationship(packageKey, relationship); + } + await this.ontologyApi.createPerspective(packageKey, conversion.perspective); + } + + private logDryRun(conversion: ConversionResult, outputToJsonFile?: boolean): void { + const payload = JSON.stringify(conversion, null, 4); + if (outputToJsonFile) { + const fileName = `${uuidv4()}_semantic_model_dry_run.json`; + fileService.writeToFileWithGivenName(payload, fileName); + logger.info(FileService.fileDownloadedMessage + fileName); + return; + } + logger.info("Dry run semantic model conversion:\n" + payload); + } +} + +export interface PushSemanticModelOptions { + poolId: string; + dataModelId: string; + packageKey: string; + schema?: string; + namespace?: string; + fromFile?: string; + dryRun?: boolean; + outputToJsonFile?: boolean; +} diff --git a/src/core/profile/profile.service.ts b/src/core/profile/profile.service.ts index 27b01846..64ade7c8 100644 --- a/src/core/profile/profile.service.ts +++ b/src/core/profile/profile.service.ts @@ -17,6 +17,7 @@ const homedir = os.homedir(); const expiryBuffer = 5000; /** All OAuth scopes; used for both device code and client credentials. */ const OAUTH_SCOPES = ["studio", "package-manager", "integration.data-pools", "action-engine.projects"]; +/** pig-sl-ontology public CRUD uses the same bearer/API-key auth as other platform services; no dedicated OAuth scope is registered in content-cli yet. */ /** Device code fallback: try without action-engine.projects if all 4 scopes fail. */ const DEVICE_CODE_SCOPES_WITHOUT_ACTION_ENGINE = ["studio", "package-manager", "integration.data-pools"]; diff --git a/tests/commands/data-model-migration/data-model-converter.service.spec.ts b/tests/commands/data-model-migration/data-model-converter.service.spec.ts new file mode 100644 index 00000000..a91673a6 --- /dev/null +++ b/tests/commands/data-model-migration/data-model-converter.service.spec.ts @@ -0,0 +1,167 @@ +import { DataModelTransport } from "../../../src/commands/data-model-migration/interfaces/data-model-transport.interfaces"; +import { DataModelConverterService } from "../../../src/commands/data-model-migration/service/data-model-converter.service"; + +const POOL_ID = "pool-123"; +const SCHEMA = "custom_schema"; + +const sampleTransport = (): DataModelTransport => ({ + id: "dm-1", + name: "Order To Cash", + poolId: POOL_ID, + dataModelType: "CASE_CENTRIC", + tables: [ + { + id: "table-orders", + name: "ORDERS", + primaryKeys: ["ORDER_ID"], + columns: [ + { name: "ORDER_ID", type: "STRING", primaryKey: true }, + { name: "CUSTOMER_ID", type: "STRING" }, + { name: "AMOUNT", type: "FLOAT" }, + ], + }, + { + id: "table-customers", + name: "CUSTOMERS", + primaryKeys: ["CUSTOMER_ID"], + columns: [ + { name: "CUSTOMER_ID", type: "STRING", primaryKey: true }, + { name: "NAME", type: "STRING" }, + ], + }, + { + id: "table-events", + name: "EVENTS", + primaryKeys: ["CASE_ID"], + columns: [ + { name: "CASE_ID", type: "STRING", primaryKey: true }, + { name: "ACTIVITY", type: "STRING" }, + { name: "EVENT_TIME", type: "DATETIME" }, + ], + }, + ], + foreignKeys: [ + { + id: "fk-1", + sourceTableId: "table-orders", + targetTableId: "table-customers", + columns: [{ sourceColumnName: "CUSTOMER_ID", targetColumnName: "CUSTOMER_ID" }], + }, + ], + processConfigurations: [ + { + activityTableId: "table-events", + caseIdColumn: "CASE_ID", + activityColumn: "ACTIVITY", + timestampColumn: "EVENT_TIME", + defaultConfiguration: true, + }, + ], +}); + +describe("DataModelConverterService", () => { + const converter = new DataModelConverterService(); + + it("Should map tables to semantic objects with ID attribute and bindings", () => { + // Arrange + const transport = sampleTransport(); + + // Act + const result = converter.convert(transport, { poolId: POOL_ID, bindingSchema: SCHEMA }); + + // Assert + const orders = result.objects.find((object) => object.key === "orders"); + expect(orders).toBeDefined(); + expect(orders?.content.attributes).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: "ID", dataType: "STRING", required: true }), + expect.objectContaining({ id: "customer_id", dataType: "STRING" }), + expect.objectContaining({ id: "amount", dataType: "DOUBLE" }), + ]) + ); + expect(orders?.content.bindings[0]).toEqual(expect.objectContaining({ + schema: SCHEMA, + table: "ORDERS", + mappingColumns: expect.arrayContaining([ + { sourceColumn: "ORDER_ID", targetColumn: "ID" }, + ]), + })); + }); + + it("Should map process configurations to semantic event sources", () => { + // Arrange + const transport = sampleTransport(); + + // Act + const result = converter.convert(transport, { poolId: POOL_ID, bindingSchema: SCHEMA }); + + // Assert + expect(result.eventSources).toHaveLength(1); + expect(result.eventSources[0].key).toBe("events_events"); + expect(result.eventSources[0].content.timestampAttribute).toBe("event_time"); + expect(result.eventSources[0].content.idAttribute).toBe("ID"); + expect(result.eventSources[0].content.bindings[0].table).toBe("EVENTS"); + }); + + it("Should map classic foreign keys to semantic relationships without junction tables", () => { + // Arrange + const transport = sampleTransport(); + + // Act + const result = converter.convert(transport, { poolId: POOL_ID, bindingSchema: SCHEMA }); + + // Assert + expect(result.relationships).toHaveLength(1); + expect(result.relationships[0].content.source).toEqual({ type: "OBJECT", referenceKey: "orders" }); + expect(result.relationships[0].content.target).toEqual({ type: "OBJECT", referenceKey: "customers" }); + expect(result.relationships[0].content.cardinality).toBe("MANY_TO_ONE"); + expect(result.relationships[0].content.foreignKeyMappings).toHaveLength(1); + expect((result.relationships[0].content as any).junctionTable).toBeUndefined(); + }); + + it("Should build a perspective referencing created objects, event sources, and relationships", () => { + // Arrange + const transport = sampleTransport(); + + // Act + const result = converter.convert(transport, { poolId: POOL_ID, bindingSchema: SCHEMA }); + + // Assert + expect(result.perspective.key).toBe("order_to_cash"); + expect(result.perspective.content.perspectiveType).toBe("CACHED"); + expect(result.perspective.content.objects).toHaveLength(3); + expect(result.perspective.content.events).toHaveLength(1); + expect(result.perspective.content.relationships).toHaveLength(1); + }); + + it("Should derive binding schema from pool id when no explicit schema is provided", () => { + // Act + const schema = DataModelConverterService.deriveBindingSchema("pool-123"); + + // Assert + expect(schema).toBe("datapipelines_pool_123_draft"); + }); + + it("Should map TIME columns to STRING because ontology has no TIME type", () => { + // Arrange + const transport: DataModelTransport = { + ...sampleTransport(), + tables: [{ + id: "table-times", + name: "TIMES", + primaryKeys: ["ID"], + columns: [{ name: "ID", type: "STRING", primaryKey: true }, { name: "START_TIME", type: "TIME" }], + }], + foreignKeys: [], + processConfigurations: [], + }; + + // Act + const result = converter.convert(transport, { poolId: POOL_ID, bindingSchema: SCHEMA }); + + // Assert + expect(result.objects[0].content.attributes).toEqual( + expect.arrayContaining([expect.objectContaining({ id: "start_time", dataType: "STRING" })]) + ); + }); +}); diff --git a/tests/commands/data-model-migration/data-model-migration.service.spec.ts b/tests/commands/data-model-migration/data-model-migration.service.spec.ts new file mode 100644 index 00000000..f4c849ce --- /dev/null +++ b/tests/commands/data-model-migration/data-model-migration.service.spec.ts @@ -0,0 +1,90 @@ +import { mockAxiosGet, mockAxiosPost, mockedAxiosInstance } from "../../utls/http-requests-mock"; +import { testContext } from "../../utls/test-context"; +import { DataModelApi } from "../../../src/commands/data-model-migration/api/data-model-api"; +import { DataModelMigrationService } from "../../../src/commands/data-model-migration/service/data-model-migration.service"; +import { loggingTestTransport } from "../../jest.setup"; + +const POOL_ID = "pool-1"; +const DATA_MODEL_ID = "dm-1"; +const PACKAGE_KEY = "my-package"; +const TRANSPORT_URL = `https://myTeam.celonis.cloud/integration/api/pools/${POOL_ID}/data-models/${DATA_MODEL_ID}/transport?includeColumns=true`; + +const transportPayload = { + id: DATA_MODEL_ID, + name: "Demo Model", + tables: [{ + id: "t1", + name: "ORDERS", + primaryKeys: ["ORDER_ID"], + columns: [{ name: "ORDER_ID", type: "STRING", primaryKey: true }], + }], + foreignKeys: [], + processConfigurations: [], +}; + +describe("Data model migration APIs", () => { + it("Should call the integration transport endpoint with includeColumns=true", async () => { + // Arrange + mockAxiosGet(TRANSPORT_URL, transportPayload); + + // Act + const result = await new DataModelApi(testContext).findOneTransport(POOL_ID, DATA_MODEL_ID, true); + + // Assert + expect(mockedAxiosInstance.get).toHaveBeenCalledWith(TRANSPORT_URL, expect.anything()); + expect(result.id).toBe(DATA_MODEL_ID); + }); +}); + +describe("DataModelMigrationService push", () => { + const objectUrl = `https://myTeam.celonis.cloud/pig-sl-ontology/api/ontology/packages/${PACKAGE_KEY}/semantic-objects`; + const perspectiveUrl = `https://myTeam.celonis.cloud/pig-sl-ontology/api/ontology/packages/${PACKAGE_KEY}/semantic-perspectives`; + + beforeEach(() => { + mockAxiosGet(TRANSPORT_URL, transportPayload); + }); + + it("Should push converted entities in dependency order during a dry run without POST calls", async () => { + // Act + await new DataModelMigrationService(testContext).pushSemanticModel({ + poolId: POOL_ID, + dataModelId: DATA_MODEL_ID, + packageKey: PACKAGE_KEY, + dryRun: true, + }); + + // Assert + expect(mockedAxiosInstance.post).not.toHaveBeenCalled(); + expect(loggingTestTransport.logMessages[0].message).toContain("Dry run semantic model conversion"); + }); + + it("Should POST semantic entities to pig-sl-ontology when dry run is disabled", async () => { + // Arrange + mockAxiosPost(objectUrl, { key: "orders" }); + mockAxiosPost( + `https://myTeam.celonis.cloud/pig-sl-ontology/api/ontology/packages/${PACKAGE_KEY}/semantic-event-sources`, + { key: "unused" } + ); + mockAxiosPost( + `https://myTeam.celonis.cloud/pig-sl-ontology/api/ontology/packages/${PACKAGE_KEY}/semantic-relationships`, + { key: "unused" } + ); + mockAxiosPost(perspectiveUrl, { key: "demo_model" }); + + // Act + await new DataModelMigrationService(testContext).pushSemanticModel({ + poolId: POOL_ID, + dataModelId: DATA_MODEL_ID, + packageKey: PACKAGE_KEY, + schema: "lake_schema", + }); + + // Assert + const postCalls = (mockedAxiosInstance.post as jest.Mock).mock.calls; + expect(postCalls[0][0]).toBe(objectUrl); + expect(JSON.parse(postCalls[0][1])).toEqual(expect.objectContaining({ key: "orders" })); + expect(postCalls[1][0]).toBe(perspectiveUrl); + expect(JSON.parse(postCalls[1][1])).toEqual(expect.objectContaining({ key: "demo_model" })); + expect(loggingTestTransport.logMessages[0].message).toContain("Successfully pushed semantic model"); + }); +}); diff --git a/tests/integration/commands/data-model-migration.spec.ts b/tests/integration/commands/data-model-migration.spec.ts new file mode 100644 index 00000000..2e83e834 --- /dev/null +++ b/tests/integration/commands/data-model-migration.spec.ts @@ -0,0 +1,52 @@ +import Module = require("../../../src/commands/data-model-migration/module"); +import { DataModelMigrationCommandService } from "../../../src/commands/data-model-migration/data-model-migration-command.service"; +import { runCli } from "../../utls/cli-runner"; + +jest.mock("../../../src/commands/data-model-migration/data-model-migration-command.service"); + +describe("data-model-migration command integration", () => { + let mockCommandService: jest.Mocked; + + beforeEach(() => { + mockCommandService = { + exportDataModel: jest.fn().mockResolvedValue(undefined), + pushSemanticModel: jest.fn().mockResolvedValue(undefined), + } as any; + (DataModelMigrationCommandService as jest.Mock).mockImplementation(() => mockCommandService); + }); + + it("Should wire export data-model to the command service", async () => { + // Act + await runCli(["export", "data-model", "--poolId", "pool-1", "--dataModelId", "dm-1"], [Module]); + + // Assert + expect(mockCommandService.exportDataModel).toHaveBeenCalledWith("pool-1", "dm-1", false); + }); + + it("Should wire push semantic-model to the command service", async () => { + // Act + await runCli([ + "push", + "semantic-model", + "--poolId", + "pool-1", + "--dataModelId", + "dm-1", + "--package", + "pkg-1", + "--dryRun", + ], [Module]); + + // Assert + expect(mockCommandService.pushSemanticModel).toHaveBeenCalledWith({ + poolId: "pool-1", + dataModelId: "dm-1", + packageKey: "pkg-1", + schema: undefined, + namespace: undefined, + fromFile: undefined, + dryRun: true, + outputToJsonFile: false, + }); + }); +}); From a849a6c9cd29a9469e198c08490bace969923b21 Mon Sep 17 00:00:00 2001 From: Philipp Hoch Date: Thu, 6 Aug 2026 13:22:08 +0200 Subject: [PATCH 2/3] Add data-model-migration commands to mkdocs nav Includes-AI-Code: true Co-authored-by: Cursor --- mkdocs.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yaml b/mkdocs.yaml index 4648ae2f..779aa9d3 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -18,6 +18,7 @@ nav: - Deployment Commands: './user-guide/deployment-commands.md' - Asset Registry Commands: './user-guide/asset-registry-commands.md' - Data Pool Commands: './user-guide/data-pool-commands.md' + - Data Model Migration Commands: './user-guide/data-model-migration-commands.md' - Action Flow Commands: './user-guide/action-flow-commands.md' - Development: - Architecture: './internal-architecture.md' From 9967876b6002c4e74cab374a5b1ac0a44590ad7e Mon Sep 17 00:00:00 2001 From: Philipp Hoch Date: Thu, 6 Aug 2026 13:30:39 +0200 Subject: [PATCH 3/3] Push semantic model via Pacman staging nodes instead of pig-sl-ontology. Aligns the migration flow with the platform CLI path (config nodes create) by posting SaveNodeTransport payloads to the package-manager API, reusing existing OAuth scopes and the standard validate/version workflow. Includes-AI-Code: true Co-authored-by: Cursor --- .../data-model-migration-commands.md | 21 +- .../data-model-migration/api/ontology-api.ts | 65 ------- .../constants/semantic-node.constants.ts | 10 + .../data-model-migration.commands.ts | 2 +- .../conversion-result.interfaces.ts | 17 +- ...faces.ts => semantic-entity.interfaces.ts} | 38 ++-- .../service/data-model-converter.service.ts | 181 +++++++++++------- .../service/data-model-migration.service.ts | 28 +-- src/core/profile/profile.service.ts | 1 - .../data-model-converter.service.spec.ts | 63 +++--- .../data-model-migration.service.spec.ts | 36 ++-- 11 files changed, 224 insertions(+), 238 deletions(-) delete mode 100644 src/commands/data-model-migration/api/ontology-api.ts create mode 100644 src/commands/data-model-migration/constants/semantic-node.constants.ts rename src/commands/data-model-migration/interfaces/{ontology.interfaces.ts => semantic-entity.interfaces.ts} (68%) diff --git a/docs/user-guide/data-model-migration-commands.md b/docs/user-guide/data-model-migration-commands.md index 03de3dc1..cc02fe9e 100644 --- a/docs/user-guide/data-model-migration-commands.md +++ b/docs/user-guide/data-model-migration-commands.md @@ -1,6 +1,6 @@ # Data Model Migration Commands -These commands export a Data Integration data model from cloud-data-integration and convert it into semantic entities in a target pig package via pig-sl-ontology. +These commands export a Data Integration data model from cloud-data-integration and convert it into semantic entity nodes in a target OCDM package via the Pacman staging-node API (same path as `config nodes create`). Supported mappings: @@ -11,7 +11,7 @@ Supported mappings: | Classic foreign key | Relationship | | Data model | Perspective | -Object-centric **object links** (`signal-links` in cloud-data-integration) are **not** supported. The converter only reads classic `foreignKeys[]` from the `/transport` export. +**Object links** (`signal-links` in cloud-data-integration) are **not** supported. The converter only reads classic `foreignKeys[]` from the `/transport` export. ## Export Data Model @@ -29,7 +29,7 @@ content-cli export data-model --poolId 80a1389d-50c5-4976-ad6e-fb5b7a2b5517 --da ## Push Semantic Model -Converts the data model and pushes semantic entities into a target pig package: +Converts the data model and pushes semantic entity nodes into a target OCDM package: ``` content-cli push semantic-model \ @@ -48,11 +48,20 @@ Options: - `--schema`: Physical lake schema used in data bindings. When omitted, the CLI derives `datapipelines__draft` (hyphens in the pool id become underscores). - `--fromFile`: Skip download and convert a previously exported transport JSON file. -- `--dryRun`: Convert only; print or write the ontology payloads without calling pig-sl-ontology. -- `--namespace`: Optional namespace for created semantic entities (defaults to ontology `"local"` when omitted). +- `--dryRun`: Convert only; print or write the Pacman node payloads without calling the staging-node API. +- `--namespace`: Optional namespace for data bindings (entity references use the package `local` namespace by default). Push order: objects → event sources → relationships → perspective. +Each entity is created as a Pacman staging node with types `SEMANTIC_OBJECT_TYPE`, `SEMANTIC_EVENT_SOURCE_TYPE`, `SEMANTIC_RELATIONSHIP_TYPE`, and `SEMANTIC_PERSPECTIVE_TYPE`. + +After pushing, validate and version the authored nodes with the standard config commands: + +``` +content-cli config package validate --packageKey --nodeKeys --layers SCHEMA BUSINESS +content-cli config versions create --packageKey --nodeFilterKeys --versionBumpOption PATCH --summaryOfChanges "..." +``` + Example dry run: ``` @@ -67,4 +76,4 @@ content-cli push semantic-model \ ## Authentication - **Download** uses the existing `integration.data-pools` OAuth scope (same as other data pool commands). -- **Push** calls `/pig-sl-ontology/api/ontology/packages/{packageKey}/semantic-*` with the profile bearer token or API key. Ensure the profile has edit access to the target package. +- **Push** uses the existing `package-manager` OAuth scope via `/pacman/api/core/staging/packages/{packageKey}/nodes`. Ensure the profile has edit access to the target package. diff --git a/src/commands/data-model-migration/api/ontology-api.ts b/src/commands/data-model-migration/api/ontology-api.ts deleted file mode 100644 index f5e87536..00000000 --- a/src/commands/data-model-migration/api/ontology-api.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { Context } from "../../../core/command/cli-context"; -import { FatalError } from "../../../core/utils/logger"; -import { HttpClient } from "../../../core/http/http-client"; -import { - OntologyNodeRequest, - OntologyNodeResponse, - SemanticEventSourceContent, - SemanticObjectContent, - SemanticPerspectiveContent, - SemanticRelationshipContent, -} from "../interfaces/ontology.interfaces"; - -const ONTOLOGY_BASE = "/pig-sl-ontology/api/ontology/packages"; - -export class OntologyApi { - - private httpClient: () => HttpClient; - - constructor(context: Context) { - this.httpClient = () => context.httpClient; - } - - /** Creates a semantic object in the target package. */ - public async createObject( - packageKey: string, - request: OntologyNodeRequest - ): Promise> { - return this.post(`${ONTOLOGY_BASE}/${packageKey}/semantic-objects`, request); - } - - /** Creates a semantic event source in the target package. */ - public async createEventSource( - packageKey: string, - request: OntologyNodeRequest - ): Promise> { - return this.post(`${ONTOLOGY_BASE}/${packageKey}/semantic-event-sources`, request); - } - - /** Creates a semantic relationship in the target package. */ - public async createRelationship( - packageKey: string, - request: OntologyNodeRequest - ): Promise> { - return this.post(`${ONTOLOGY_BASE}/${packageKey}/semantic-relationships`, request); - } - - /** Creates a semantic perspective in the target package. */ - public async createPerspective( - packageKey: string, - request: OntologyNodeRequest - ): Promise> { - return this.post(`${ONTOLOGY_BASE}/${packageKey}/semantic-perspectives`, request); - } - - private post( - url: string, - request: OntologyNodeRequest - ): Promise { - return this.httpClient() - .post(url, request) - .catch((error) => { - throw new FatalError(`Problem creating semantic entity '${request.key}': ${error}`); - }); - } -} diff --git a/src/commands/data-model-migration/constants/semantic-node.constants.ts b/src/commands/data-model-migration/constants/semantic-node.constants.ts new file mode 100644 index 00000000..d2d3898e --- /dev/null +++ b/src/commands/data-model-migration/constants/semantic-node.constants.ts @@ -0,0 +1,10 @@ +/** Pacman node type identifiers for context-model semantic assets. */ +export const SEMANTIC_NODE_TYPES = { + OBJECT: "SEMANTIC_OBJECT_TYPE", + EVENT_SOURCE: "SEMANTIC_EVENT_SOURCE_TYPE", + RELATIONSHIP: "SEMANTIC_RELATIONSHIP_TYPE", + PERSPECTIVE: "SEMANTIC_PERSPECTIVE_TYPE", +} as const; + +/** Asset-registry schema version for semantic entity node types. */ +export const SEMANTIC_SCHEMA_VERSION = 1; diff --git a/src/commands/data-model-migration/data-model-migration.commands.ts b/src/commands/data-model-migration/data-model-migration.commands.ts index bfdbab58..7d5e1958 100644 --- a/src/commands/data-model-migration/data-model-migration.commands.ts +++ b/src/commands/data-model-migration/data-model-migration.commands.ts @@ -23,7 +23,7 @@ export class DataModelMigrationCommands { .option("--schema ", "Physical lake schema for data bindings (overrides pool-derived default)") .option("--namespace ", "Namespace for created semantic entities") .option("-f, --fromFile ", "Use a previously exported data model transport JSON file") - .option("--dryRun", "Convert only; print or write payloads without pushing to ontology") + .option("--dryRun", "Convert only; print or write node payloads without pushing to Pacman") .option("--outputToJsonFile", "With --dryRun, write conversion output to a JSON file") .action(this.pushSemanticModel); } diff --git a/src/commands/data-model-migration/interfaces/conversion-result.interfaces.ts b/src/commands/data-model-migration/interfaces/conversion-result.interfaces.ts index 0130aba3..48977ea7 100644 --- a/src/commands/data-model-migration/interfaces/conversion-result.interfaces.ts +++ b/src/commands/data-model-migration/interfaces/conversion-result.interfaces.ts @@ -1,20 +1,15 @@ -import { - OntologyNodeRequest, - SemanticEventSourceContent, - SemanticObjectContent, - SemanticPerspectiveContent, - SemanticRelationshipContent, -} from "./ontology.interfaces"; +import { SaveNodeTransport } from "../../configuration-management/interfaces/node.interfaces"; export interface ConversionResult { - objects: OntologyNodeRequest[]; - eventSources: OntologyNodeRequest[]; - relationships: OntologyNodeRequest[]; - perspective: OntologyNodeRequest; + objects: SaveNodeTransport[]; + eventSources: SaveNodeTransport[]; + relationships: SaveNodeTransport[]; + perspective: SaveNodeTransport; } export interface ConversionOptions { poolId: string; bindingSchema: string; + packageKey: string; namespace?: string; } diff --git a/src/commands/data-model-migration/interfaces/ontology.interfaces.ts b/src/commands/data-model-migration/interfaces/semantic-entity.interfaces.ts similarity index 68% rename from src/commands/data-model-migration/interfaces/ontology.interfaces.ts rename to src/commands/data-model-migration/interfaces/semantic-entity.interfaces.ts index 7a6b0a8b..8b0d2a39 100644 --- a/src/commands/data-model-migration/interfaces/ontology.interfaces.ts +++ b/src/commands/data-model-migration/interfaces/semantic-entity.interfaces.ts @@ -35,18 +35,21 @@ export interface Binding { mappingColumns: MappingColumn[]; } -export interface SemanticObjectContent { +export interface SemanticObjectConfiguration { + active: boolean; attributes: OntologyAttribute[]; bindings: Binding[]; - primaryKeys?: string[]; + primaryKeys: string[]; + calculatedAttributes: []; } -export interface SemanticEventSourceContent { +export interface SemanticEventSourceConfiguration { + active: boolean; attributes: OntologyAttribute[]; bindings: Binding[]; + primaryKeys: string[]; timestampAttribute: string; idAttribute: string; - primaryKeys?: string[]; } export interface Reference { @@ -60,32 +63,19 @@ export interface ForeignKeyMapping { targetField: OntologyAttribute; } -export interface SemanticRelationshipContent { +export interface SemanticRelationshipConfiguration { source: Reference; target: Reference; - relationshipType?: RelationshipType; - cardinality?: Cardinality; + relationshipType: RelationshipType; + cardinality: Cardinality; foreignKeyMappings: ForeignKeyMapping[]; } -export interface SemanticPerspectiveContent { +export interface SemanticPerspectiveConfiguration { + active: boolean; objects: Reference[]; events: Reference[]; relationships: Reference[]; - perspectiveType?: PerspectiveType; - INSTANTIATE_ALL_EVENTS?: boolean; -} - -export interface OntologyNodeRequest { - key: string; - name: string; - namespace?: string; - content: T; -} - -export interface OntologyNodeResponse { - key: string; - name: string; - packageNodeKey?: string; - content?: T; + perspectiveType: PerspectiveType; + INSTANTIATE_ALL_EVENTS: boolean; } diff --git a/src/commands/data-model-migration/service/data-model-converter.service.ts b/src/commands/data-model-migration/service/data-model-converter.service.ts index 6c65ca1c..5cae7716 100644 --- a/src/commands/data-model-migration/service/data-model-converter.service.ts +++ b/src/commands/data-model-migration/service/data-model-converter.service.ts @@ -1,3 +1,5 @@ +import { SaveNodeTransport, NodeConfiguration } from "../../configuration-management/interfaces/node.interfaces"; +import { SEMANTIC_NODE_TYPES, SEMANTIC_SCHEMA_VERSION } from "../constants/semantic-node.constants"; import { ColumnType, DataModelConfigurationTransport, @@ -11,13 +13,12 @@ import { Binding, ForeignKeyMapping, OntologyAttribute, - OntologyNodeRequest, Reference, - SemanticEventSourceContent, - SemanticObjectContent, - SemanticPerspectiveContent, - SemanticRelationshipContent, -} from "../interfaces/ontology.interfaces"; + SemanticEventSourceConfiguration, + SemanticObjectConfiguration, + SemanticPerspectiveConfiguration, + SemanticRelationshipConfiguration, +} from "../interfaces/semantic-entity.interfaces"; interface TableContext { table: DataModelTableTransport; @@ -26,20 +27,20 @@ interface TableContext { attributeById: Map; } -/** Converts a Data Integration data model transport into pig semantic entity requests. */ +/** Converts a Data Integration data model transport into Pacman staging node payloads. */ export class DataModelConverterService { - /** Converts tables, process configurations, and classic foreign keys into semantic entities. */ + /** Converts tables, process configurations, and classic foreign keys into semantic entity nodes. */ public convert(transport: DataModelTransport, options: ConversionOptions): ConversionResult { const tableContexts = this.buildTableContexts(transport.tables ?? []); const objects = tableContexts.map((ctx) => this.convertTable(ctx, options)); const eventSources = (transport.processConfigurations ?? []) .map((config) => this.convertProcessConfiguration(config, tableContexts, options)) - .filter((request): request is OntologyNodeRequest => request !== null); + .filter((node): node is SaveNodeTransport => node !== null); const relationships = (transport.foreignKeys ?? []) - .map((fk) => this.convertForeignKey(fk, tableContexts)) - .filter((request): request is OntologyNodeRequest => request !== null); - const perspective = this.convertPerspective(transport, objects, eventSources, relationships); + .map((fk) => this.convertForeignKey(fk, tableContexts, options)) + .filter((node): node is SaveNodeTransport => node !== null); + const perspective = this.convertPerspective(transport, objects, eventSources, relationships, options); return { objects, eventSources, relationships, perspective }; } @@ -76,30 +77,31 @@ export class DataModelConverterService { }); } - private convertTable( - ctx: TableContext, - options: ConversionOptions - ): OntologyNodeRequest { + private convertTable(ctx: TableContext, options: ConversionOptions): SaveNodeTransport { const attributes = Array.from(ctx.attributeById.values()); const primaryKeys = attributes.some((attribute) => attribute.id === "ID") ? ["ID"] : []; - - return { - key: ctx.objectKey, - name: ctx.table.name, - namespace: options.namespace, - content: { - attributes, - primaryKeys, - bindings: [this.buildTableBinding(ctx, options.bindingSchema)], - }, + const configuration: SemanticObjectConfiguration = { + active: true, + attributes, + primaryKeys, + calculatedAttributes: [], + bindings: [this.buildTableBinding(ctx, options)], }; + + return buildSemanticNode( + options.packageKey, + ctx.objectKey, + ctx.table.name, + SEMANTIC_NODE_TYPES.OBJECT, + configuration + ); } private convertProcessConfiguration( config: DataModelConfigurationTransport, tableContexts: TableContext[], options: ConversionOptions - ): OntologyNodeRequest | null { + ): SaveNodeTransport | null { const activityTable = tableContexts.find((ctx) => ctx.table.id === config.activityTableId); if (!activityTable) { return null; @@ -147,30 +149,35 @@ export class DataModelConverterService { targetColumn: columnToAttributeId.get(columnName) ?? sanitizeKey(columnName), })); - return { - key: eventSourceKey, - name: `${activityTable.table.name} Events`, - namespace: options.namespace, - content: { - attributes: Array.from(attributeById.values()), - primaryKeys: ["ID"], - timestampAttribute, - idAttribute: "ID", - bindings: [{ - name: `${eventSourceKey}-binding`, - namespace: options.namespace, - schema: options.bindingSchema, - table: activityTable.table.name, - mappingColumns, - }], - }, + const configuration: SemanticEventSourceConfiguration = { + active: true, + attributes: Array.from(attributeById.values()), + primaryKeys: ["ID"], + timestampAttribute, + idAttribute: "ID", + bindings: [{ + name: `${eventSourceKey}-binding`, + namespace: options.namespace, + schema: options.bindingSchema, + table: activityTable.table.name, + mappingColumns, + }], }; + + return buildSemanticNode( + options.packageKey, + eventSourceKey, + `${activityTable.table.name} Events`, + SEMANTIC_NODE_TYPES.EVENT_SOURCE, + configuration + ); } private convertForeignKey( foreignKey: DataModelForeignKeyTransport, - tableContexts: TableContext[] - ): OntologyNodeRequest | null { + tableContexts: TableContext[], + options: ConversionOptions + ): SaveNodeTransport | null { const source = tableContexts.find((ctx) => ctx.table.id === foreignKey.sourceTableId); const target = tableContexts.find((ctx) => ctx.table.id === foreignKey.targetTableId); if (!source || !target) { @@ -183,42 +190,50 @@ export class DataModelConverterService { })); const relationshipKey = sanitizeKey(`rel-${source.objectKey}-${target.objectKey}-${foreignKey.id}`); - - return { - key: relationshipKey, - name: `${source.table.name} -> ${target.table.name}`, - content: { - source: objectReference(source.objectKey), - target: objectReference(target.objectKey), - relationshipType: "INSTANCE_TO_INSTANCE", - cardinality: "MANY_TO_ONE", - foreignKeyMappings, - }, + const configuration: SemanticRelationshipConfiguration = { + source: objectReference(source.objectKey), + target: objectReference(target.objectKey), + relationshipType: "INSTANCE_TO_INSTANCE", + cardinality: "MANY_TO_ONE", + foreignKeyMappings, }; + + return buildSemanticNode( + options.packageKey, + relationshipKey, + `${source.table.name} -> ${target.table.name}`, + SEMANTIC_NODE_TYPES.RELATIONSHIP, + configuration + ); } private convertPerspective( transport: DataModelTransport, - objects: OntologyNodeRequest[], - eventSources: OntologyNodeRequest[], - relationships: OntologyNodeRequest[] - ): OntologyNodeRequest { + objects: SaveNodeTransport[], + eventSources: SaveNodeTransport[], + relationships: SaveNodeTransport[], + options: ConversionOptions + ): SaveNodeTransport { const perspectiveKey = sanitizeKey(transport.name || transport.id); - - return { - key: perspectiveKey, - name: transport.name, - content: { - objects: objects.map((object) => objectReference(object.key)), - events: eventSources.map((eventSource) => eventSourceReference(eventSource.key)), - relationships: relationships.map((relationship) => relationshipReference(relationship.key)), - perspectiveType: "CACHED", - INSTANTIATE_ALL_EVENTS: false, - }, + const configuration: SemanticPerspectiveConfiguration = { + active: true, + objects: objects.map((object) => objectReference(object.key)), + events: eventSources.map((eventSource) => eventSourceReference(eventSource.key)), + relationships: relationships.map((relationship) => relationshipReference(relationship.key)), + perspectiveType: "CACHED", + INSTANTIATE_ALL_EVENTS: false, }; + + return buildSemanticNode( + options.packageKey, + perspectiveKey, + transport.name, + SEMANTIC_NODE_TYPES.PERSPECTIVE, + configuration + ); } - private buildTableBinding(ctx: TableContext, schema: string): Binding { + private buildTableBinding(ctx: TableContext, options: ConversionOptions): Binding { const mappingColumns = (ctx.table.columns ?? []).map((column) => ({ sourceColumn: column.name, targetColumn: ctx.columnToAttributeId.get(column.name) ?? sanitizeKey(column.name), @@ -226,7 +241,8 @@ export class DataModelConverterService { return { name: `${ctx.objectKey}-binding`, - schema, + namespace: options.namespace, + schema: options.bindingSchema, table: ctx.table.name, mappingColumns, }; @@ -272,6 +288,23 @@ export class DataModelConverterService { } } +function buildSemanticNode( + packageKey: string, + key: string, + name: string, + type: string, + configuration: NodeConfiguration +): SaveNodeTransport { + return { + key, + name, + type, + parentNodeKey: packageKey, + schemaVersion: SEMANTIC_SCHEMA_VERSION, + configuration, + }; +} + function objectReference(referenceKey: string): Reference { return { type: "OBJECT", referenceKey }; } diff --git a/src/commands/data-model-migration/service/data-model-migration.service.ts b/src/commands/data-model-migration/service/data-model-migration.service.ts index bab0c0fe..0b4fd3f5 100644 --- a/src/commands/data-model-migration/service/data-model-migration.service.ts +++ b/src/commands/data-model-migration/service/data-model-migration.service.ts @@ -1,9 +1,10 @@ import { v4 as uuidv4 } from "uuid"; +import { NodeApi } from "../../configuration-management/api/node-api"; +import { SaveNodeTransport } from "../../configuration-management/interfaces/node.interfaces"; import { FileService, fileService } from "../../../core/utils/file-service"; import { logger } from "../../../core/utils/logger"; import { Context } from "../../../core/command/cli-context"; import { DataModelApi } from "../api/data-model-api"; -import { OntologyApi } from "../api/ontology-api"; import { DataModelTransport } from "../interfaces/data-model-transport.interfaces"; import { ConversionResult } from "../interfaces/conversion-result.interfaces"; import { DataModelConverterService } from "./data-model-converter.service"; @@ -11,12 +12,12 @@ import { DataModelConverterService } from "./data-model-converter.service"; export class DataModelMigrationService { private dataModelApi: DataModelApi; - private ontologyApi: OntologyApi; + private nodeApi: NodeApi; private converter: DataModelConverterService; constructor(context: Context) { this.dataModelApi = new DataModelApi(context); - this.ontologyApi = new OntologyApi(context); + this.nodeApi = new NodeApi(context); this.converter = new DataModelConverterService(); } @@ -35,13 +36,14 @@ export class DataModelMigrationService { logger.info("Exported Data Model:\n" + payload); } - /** Converts a data model transport and pushes semantic entities into a target package. */ + /** Converts a data model transport and pushes semantic entity nodes into a target package. */ public async pushSemanticModel(options: PushSemanticModelOptions): Promise { const transport = await this.loadTransport(options); const bindingSchema = DataModelConverterService.deriveBindingSchema(options.poolId, options.schema); const conversion = this.converter.convert(transport, { poolId: options.poolId, bindingSchema, + packageKey: options.packageKey, namespace: options.namespace, }); @@ -68,16 +70,16 @@ export class DataModelMigrationService { } private async pushConversion(packageKey: string, conversion: ConversionResult): Promise { - for (const object of conversion.objects) { - await this.ontologyApi.createObject(packageKey, object); - } - for (const eventSource of conversion.eventSources) { - await this.ontologyApi.createEventSource(packageKey, eventSource); - } - for (const relationship of conversion.relationships) { - await this.ontologyApi.createRelationship(packageKey, relationship); + const nodes: SaveNodeTransport[] = [ + ...conversion.objects, + ...conversion.eventSources, + ...conversion.relationships, + conversion.perspective, + ]; + + for (const node of nodes) { + await this.nodeApi.createStagingNode(packageKey, node, false); } - await this.ontologyApi.createPerspective(packageKey, conversion.perspective); } private logDryRun(conversion: ConversionResult, outputToJsonFile?: boolean): void { diff --git a/src/core/profile/profile.service.ts b/src/core/profile/profile.service.ts index 64ade7c8..27b01846 100644 --- a/src/core/profile/profile.service.ts +++ b/src/core/profile/profile.service.ts @@ -17,7 +17,6 @@ const homedir = os.homedir(); const expiryBuffer = 5000; /** All OAuth scopes; used for both device code and client credentials. */ const OAUTH_SCOPES = ["studio", "package-manager", "integration.data-pools", "action-engine.projects"]; -/** pig-sl-ontology public CRUD uses the same bearer/API-key auth as other platform services; no dedicated OAuth scope is registered in content-cli yet. */ /** Device code fallback: try without action-engine.projects if all 4 scopes fail. */ const DEVICE_CODE_SCOPES_WITHOUT_ACTION_ENGINE = ["studio", "package-manager", "integration.data-pools"]; diff --git a/tests/commands/data-model-migration/data-model-converter.service.spec.ts b/tests/commands/data-model-migration/data-model-converter.service.spec.ts index a91673a6..35c76aef 100644 --- a/tests/commands/data-model-migration/data-model-converter.service.spec.ts +++ b/tests/commands/data-model-migration/data-model-converter.service.spec.ts @@ -2,6 +2,7 @@ import { DataModelTransport } from "../../../src/commands/data-model-migration/i import { DataModelConverterService } from "../../../src/commands/data-model-migration/service/data-model-converter.service"; const POOL_ID = "pool-123"; +const PACKAGE_KEY = "my-package"; const SCHEMA = "custom_schema"; const sampleTransport = (): DataModelTransport => ({ @@ -59,27 +60,36 @@ const sampleTransport = (): DataModelTransport => ({ ], }); +const conversionOptions = () => ({ + poolId: POOL_ID, + bindingSchema: SCHEMA, + packageKey: PACKAGE_KEY, +}); + describe("DataModelConverterService", () => { const converter = new DataModelConverterService(); - it("Should map tables to semantic objects with ID attribute and bindings", () => { + it("Should map tables to semantic object nodes with ID attribute and bindings", () => { // Arrange const transport = sampleTransport(); // Act - const result = converter.convert(transport, { poolId: POOL_ID, bindingSchema: SCHEMA }); + const result = converter.convert(transport, conversionOptions()); // Assert const orders = result.objects.find((object) => object.key === "orders"); expect(orders).toBeDefined(); - expect(orders?.content.attributes).toEqual( + expect(orders?.type).toBe("SEMANTIC_OBJECT_TYPE"); + expect(orders?.parentNodeKey).toBe(PACKAGE_KEY); + expect(orders?.schemaVersion).toBe(1); + expect(orders?.configuration?.attributes).toEqual( expect.arrayContaining([ expect.objectContaining({ id: "ID", dataType: "STRING", required: true }), expect.objectContaining({ id: "customer_id", dataType: "STRING" }), expect.objectContaining({ id: "amount", dataType: "DOUBLE" }), ]) ); - expect(orders?.content.bindings[0]).toEqual(expect.objectContaining({ + expect(orders?.configuration?.bindings[0]).toEqual(expect.objectContaining({ schema: SCHEMA, table: "ORDERS", mappingColumns: expect.arrayContaining([ @@ -88,50 +98,53 @@ describe("DataModelConverterService", () => { })); }); - it("Should map process configurations to semantic event sources", () => { + it("Should map process configurations to semantic event source nodes", () => { // Arrange const transport = sampleTransport(); // Act - const result = converter.convert(transport, { poolId: POOL_ID, bindingSchema: SCHEMA }); + const result = converter.convert(transport, conversionOptions()); // Assert expect(result.eventSources).toHaveLength(1); expect(result.eventSources[0].key).toBe("events_events"); - expect(result.eventSources[0].content.timestampAttribute).toBe("event_time"); - expect(result.eventSources[0].content.idAttribute).toBe("ID"); - expect(result.eventSources[0].content.bindings[0].table).toBe("EVENTS"); + expect(result.eventSources[0].type).toBe("SEMANTIC_EVENT_SOURCE_TYPE"); + expect(result.eventSources[0].configuration?.timestampAttribute).toBe("event_time"); + expect(result.eventSources[0].configuration?.idAttribute).toBe("ID"); + expect(result.eventSources[0].configuration?.bindings[0].table).toBe("EVENTS"); }); - it("Should map classic foreign keys to semantic relationships without junction tables", () => { + it("Should map classic foreign keys to semantic relationship nodes without junction tables", () => { // Arrange const transport = sampleTransport(); // Act - const result = converter.convert(transport, { poolId: POOL_ID, bindingSchema: SCHEMA }); + const result = converter.convert(transport, conversionOptions()); // Assert expect(result.relationships).toHaveLength(1); - expect(result.relationships[0].content.source).toEqual({ type: "OBJECT", referenceKey: "orders" }); - expect(result.relationships[0].content.target).toEqual({ type: "OBJECT", referenceKey: "customers" }); - expect(result.relationships[0].content.cardinality).toBe("MANY_TO_ONE"); - expect(result.relationships[0].content.foreignKeyMappings).toHaveLength(1); - expect((result.relationships[0].content as any).junctionTable).toBeUndefined(); + expect(result.relationships[0].type).toBe("SEMANTIC_RELATIONSHIP_TYPE"); + expect(result.relationships[0].configuration?.source).toEqual({ type: "OBJECT", referenceKey: "orders" }); + expect(result.relationships[0].configuration?.target).toEqual({ type: "OBJECT", referenceKey: "customers" }); + expect(result.relationships[0].configuration?.cardinality).toBe("MANY_TO_ONE"); + expect(result.relationships[0].configuration?.foreignKeyMappings).toHaveLength(1); + expect(result.relationships[0].configuration?.junctionTable).toBeUndefined(); }); - it("Should build a perspective referencing created objects, event sources, and relationships", () => { + it("Should build a perspective node referencing created objects, event sources, and relationships", () => { // Arrange const transport = sampleTransport(); // Act - const result = converter.convert(transport, { poolId: POOL_ID, bindingSchema: SCHEMA }); + const result = converter.convert(transport, conversionOptions()); // Assert expect(result.perspective.key).toBe("order_to_cash"); - expect(result.perspective.content.perspectiveType).toBe("CACHED"); - expect(result.perspective.content.objects).toHaveLength(3); - expect(result.perspective.content.events).toHaveLength(1); - expect(result.perspective.content.relationships).toHaveLength(1); + expect(result.perspective.type).toBe("SEMANTIC_PERSPECTIVE_TYPE"); + expect(result.perspective.configuration?.perspectiveType).toBe("CACHED"); + expect(result.perspective.configuration?.objects).toHaveLength(3); + expect(result.perspective.configuration?.events).toHaveLength(1); + expect(result.perspective.configuration?.relationships).toHaveLength(1); }); it("Should derive binding schema from pool id when no explicit schema is provided", () => { @@ -142,7 +155,7 @@ describe("DataModelConverterService", () => { expect(schema).toBe("datapipelines_pool_123_draft"); }); - it("Should map TIME columns to STRING because ontology has no TIME type", () => { + it("Should map TIME columns to STRING because semantic schemas have no TIME type", () => { // Arrange const transport: DataModelTransport = { ...sampleTransport(), @@ -157,10 +170,10 @@ describe("DataModelConverterService", () => { }; // Act - const result = converter.convert(transport, { poolId: POOL_ID, bindingSchema: SCHEMA }); + const result = converter.convert(transport, conversionOptions()); // Assert - expect(result.objects[0].content.attributes).toEqual( + expect(result.objects[0].configuration?.attributes).toEqual( expect.arrayContaining([expect.objectContaining({ id: "start_time", dataType: "STRING" })]) ); }); diff --git a/tests/commands/data-model-migration/data-model-migration.service.spec.ts b/tests/commands/data-model-migration/data-model-migration.service.spec.ts index f4c849ce..789b1ba0 100644 --- a/tests/commands/data-model-migration/data-model-migration.service.spec.ts +++ b/tests/commands/data-model-migration/data-model-migration.service.spec.ts @@ -8,6 +8,7 @@ const POOL_ID = "pool-1"; const DATA_MODEL_ID = "dm-1"; const PACKAGE_KEY = "my-package"; const TRANSPORT_URL = `https://myTeam.celonis.cloud/integration/api/pools/${POOL_ID}/data-models/${DATA_MODEL_ID}/transport?includeColumns=true`; +const NODE_CREATE_URL = `https://myTeam.celonis.cloud/pacman/api/core/staging/packages/${PACKAGE_KEY}/nodes`; const transportPayload = { id: DATA_MODEL_ID, @@ -37,9 +38,6 @@ describe("Data model migration APIs", () => { }); describe("DataModelMigrationService push", () => { - const objectUrl = `https://myTeam.celonis.cloud/pig-sl-ontology/api/ontology/packages/${PACKAGE_KEY}/semantic-objects`; - const perspectiveUrl = `https://myTeam.celonis.cloud/pig-sl-ontology/api/ontology/packages/${PACKAGE_KEY}/semantic-perspectives`; - beforeEach(() => { mockAxiosGet(TRANSPORT_URL, transportPayload); }); @@ -58,18 +56,10 @@ describe("DataModelMigrationService push", () => { expect(loggingTestTransport.logMessages[0].message).toContain("Dry run semantic model conversion"); }); - it("Should POST semantic entities to pig-sl-ontology when dry run is disabled", async () => { + it("Should POST staging nodes to Pacman when dry run is disabled", async () => { // Arrange - mockAxiosPost(objectUrl, { key: "orders" }); - mockAxiosPost( - `https://myTeam.celonis.cloud/pig-sl-ontology/api/ontology/packages/${PACKAGE_KEY}/semantic-event-sources`, - { key: "unused" } - ); - mockAxiosPost( - `https://myTeam.celonis.cloud/pig-sl-ontology/api/ontology/packages/${PACKAGE_KEY}/semantic-relationships`, - { key: "unused" } - ); - mockAxiosPost(perspectiveUrl, { key: "demo_model" }); + mockAxiosPost(NODE_CREATE_URL, { key: "orders" }); + mockAxiosPost(NODE_CREATE_URL, { key: "demo_model" }); // Act await new DataModelMigrationService(testContext).pushSemanticModel({ @@ -81,10 +71,20 @@ describe("DataModelMigrationService push", () => { // Assert const postCalls = (mockedAxiosInstance.post as jest.Mock).mock.calls; - expect(postCalls[0][0]).toBe(objectUrl); - expect(JSON.parse(postCalls[0][1])).toEqual(expect.objectContaining({ key: "orders" })); - expect(postCalls[1][0]).toBe(perspectiveUrl); - expect(JSON.parse(postCalls[1][1])).toEqual(expect.objectContaining({ key: "demo_model" })); + expect(postCalls).toHaveLength(2); + expect(postCalls[0][0]).toBe(NODE_CREATE_URL); + expect(JSON.parse(postCalls[0][1])).toEqual(expect.objectContaining({ + key: "orders", + type: "SEMANTIC_OBJECT_TYPE", + parentNodeKey: PACKAGE_KEY, + schemaVersion: 1, + })); + expect(postCalls[1][0]).toBe(NODE_CREATE_URL); + expect(JSON.parse(postCalls[1][1])).toEqual(expect.objectContaining({ + key: "demo_model", + type: "SEMANTIC_PERSPECTIVE_TYPE", + parentNodeKey: PACKAGE_KEY, + })); expect(loggingTestTransport.logMessages[0].message).toContain("Successfully pushed semantic model"); }); });