diff --git a/.gitignore b/.gitignore index 4104bfe..2ce6852 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ gen/ package-lock.json .env .cdsrc-private.json -resources/ \ No newline at end of file +resources/ +.cds/models/ diff --git a/CHANGELOG.md b/CHANGELOG.md index a15e27c..36480ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,11 @@ ### Added -- Add the `ai-sqlite` kind with a `VECTOR_EMBEDDING` function using ONNX Runtime and the `Xenova/all-MiniLM-L6-v2` model (384 dimensions) - - Downloads the pinned model revision on-demand from Hugging Face (~91MB), verifies its size and SHA-256, and caches it locally +- Add the `ai-sqlite` kind with a `VECTOR_EMBEDDING` function using compatible ONNX encoder models + - Requires `cds.env.requires.db.embedding.model`; automatically discovers model metadata and supports warned, on-demand provisioning into `.cds/models` + - Adds `npx @cap-js/ai install-model ` with an optional shared model-cache root - Uses `@huggingface/tokenizers` and chunks long input without dropping per-chunk special tokens - - Allows an explicit, checksum-verified compatible encoder model descriptor per `ai-sqlite` service + - Configures embedding runtimes only through `model` and an optional relative, absolute, or home-relative `directory`; discovered metadata remains in the provisioned lock - Supports both 3-parameter `(text, text_type, model_and_version)` and 4-parameter variants with `remote_source` - Compatible with `SAP_GXY.20250407` and `SAP_GXY.20240715` model versions - Synchronous execution suitable for SQLite user-defined functions diff --git a/README.md b/README.md index 2f2de63..a7daa3f 100644 --- a/README.md +++ b/README.md @@ -207,7 +207,7 @@ resources: ### 3. Local Vector Embeddings with SQLite -The `ai-sqlite` database kind extends `@cap-js/sqlite` with local semantic embeddings using an ONNX encoder model. It uses the pinned `Xenova/all-MiniLM-L6-v2` model by default. +The `ai-sqlite` database kind extends `@cap-js/sqlite` with local semantic embeddings using an ONNX encoder model. Configure the model explicitly for every service. #### Usage @@ -219,18 +219,67 @@ npm add @cap-js/sqlite onnxruntime-node@1.20.1 `ai-sqlite` currently requires exactly `onnxruntime-node` 1.20.1 because synchronous SQLite functions need a version-specific native runtime API. -Select `ai-sqlite` for the database service: +#### Model provisioning + +Runtime configuration is intentionally limited to a model name and an optional model-cache root: + +```json +{ + "cds": { + "requires": { + "db": { + "kind": "ai-sqlite", + "embedding": { + "model": "foo/bar" + } + } + } + } +} +``` + +`embedding.model` is required. If it is absent, `ai-sqlite` fails during startup. No revision, dimensions, tokenizer, file, pooling, checksum, or descriptor settings are accepted in runtime configuration. + +Without `directory`, the model is stored below the CAP project at `.cds/models/foo/bar`. Startup reuses a valid installation from there. If it is missing, startup logs a warning, discovers and downloads the model, generates `embedding.lock.json`, and reuses that installation on subsequent starts. + +To provision the project-local model before startup instead: + +```sh +npx @cap-js/ai install-model foo/bar +``` + +To share a model across projects, select another cache root: + +```sh +npx @cap-js/ai install-model foo/bar --directory ~/.cds/models +``` ```json { "cds": { "requires": { - "db": "ai-sqlite" + "db": { + "kind": "ai-sqlite", + "embedding": { + "model": "foo/bar", + "directory": "~/.cds/models" + } + } } } } ``` +`directory` always names the cache root; the model is stored below it using the repository path, for example `~/.cds/models/foo/bar`. Relative directories are resolved from `cds.root`, absolute directories are used unchanged, and `~/` is resolved from the user's home directory. + +When `directory` is configured, startup treats it as a pre-installed shared cache: it verifies the model but does not download or modify it. This makes runtime deployment deterministic and allows the shared directory to be read-only. + +##### Automatic model discovery + +The installer resolves the model's current Hugging Face revision to an immutable commit, selects the conventional `onnx/model.onnx` and tokenizer/configuration files, calculates or obtains their checksums, and derives the dimensions, tokenizer limit, pooling, and normalization metadata. It then writes all resolved metadata to `embedding.lock.json` alongside the downloaded artifacts. + +Discovery supports compatible Hugging Face ONNX Sentence Transformers models with machine-readable pooling semantics. Repositories with missing or ambiguous artifacts or semantics fail with a compatibility error instead of using guessed defaults. Once installed, startup uses the pinned lock and does not follow later changes to the model repository. + The HANA-compatible SQL function can then be used in CQL: ```js @@ -247,73 +296,33 @@ SELECT.from('Books').columns` **Returns:** -- JSON stringified array of embedding values (384 dimensions for the default MiniLM model; custom models use their configured `dimensions`) +- JSON stringified array of embedding values with the configured model's dimensions **Features:** - **Initialization**: The ONNX model is loaded when the `ai-sqlite` service starts -- **Verified cache**: The pinned model revision and artifact set are cached by default below the user's data directory; set `CDS_AI_MODEL_CACHE` to use a pre-provisioned cache root +- **Automatic provisioning**: Use model-only configuration for warned, on-demand installation into `.cds/models` +- **Explicit provisioning**: Preinstall local or shared models with `npx @cap-js/ai install-model` +- **Verified artifacts**: The provisioned lock pins the revision, artifact sizes, and SHA-256 checksums - **Hugging Face tokenization**: Uses `@huggingface/tokenizers` and safely chunks text that exceeds the model limit - **Deterministic**: Same input always produces same output -- **Normalized vectors**: MiniLM embeddings are L2-normalized; custom descriptors control this with `output.normalize` +- **Automatic output handling**: Pooling and normalization are derived from Sentence Transformers metadata - **Semantic similarity**: Embeddings capture text meaning for similarity search -#### Compatible custom encoder models +#### Compatible encoder models -Configure a different model through the database service's `embedding` option. Models are not discovered dynamically: every artifact must belong to an immutable revision and have an expected size and SHA-256 checksum. +Compatible repositories must provide `onnx/model.onnx`, `tokenizer.json`, `tokenizer_config.json`, and `config.json`. The model must accept `input_ids` and may additionally accept `attention_mask` and `token_type_ids`, all as `int64` tensors, and expose a `last_hidden_state` float output. -```json -{ - "cds": { - "requires": { - "db": { - "kind": "ai-sqlite", - "embedding": { - "repository": "organization/model", - "revision": "0123456789abcdef0123456789abcdef01234567", - "dimensions": 768, - "maxLength": 512, - "files": [ - { - "role": "model", - "name": "model.onnx", - "path": "onnx/model.onnx", - "size": 123456789, - "sha256": "<64 lowercase hexadecimal characters>" - }, - { - "role": "tokenizer", - "name": "tokenizer.json", - "path": "tokenizer.json", - "size": 123456, - "sha256": "<64 lowercase hexadecimal characters>" - }, - { - "role": "tokenizerConfig", - "name": "tokenizer_config.json", - "path": "tokenizer_config.json", - "size": 1234, - "sha256": "<64 lowercase hexadecimal characters>" - } - ], - "output": { - "name": "last_hidden_state", - "pooling": "mean", - "normalize": true - } - } - } - } - } -} -``` +Pooling semantics are read from Sentence Transformers `modules.json` and its pooling configuration. Converted repositories such as `Xenova/*` can declare a single `base_model`; its immutable Sentence Transformers metadata is used to determine mean or CLS pooling and normalization. Unsupported module chains, ambiguous pooling modes, missing metadata, or incompatible ONNX inputs and outputs fail explicitly. -Compatible models must accept `input_ids` and may additionally accept `attention_mask` and `token_type_ids`, all as `int64` tensors. Their configured float32 or float64 output must support `mean` or `cls` pooling from `[1, sequence, dimensions]`, or `none` for an already pooled `[dimensions]` or `[1, dimensions]` tensor. Additional pinned ONNX data files can use the `auxiliary` role. Startup probes the model and rejects incompatible input names, output names, types, shapes, or dimensions. +Provisioning canonicalizes symlinked parent directories and rejects a model directory that is itself a symlink. Existing valid locks remain pinned and are reused rather than silently following changes to the repository's default branch. **Error Handling:** -- Starting `ai-sqlite` fails if the ONNX model cannot be initialized -- Downloads are time-limited and accepted only when their expected size and SHA-256 match +- Starting `ai-sqlite` fails if `cds.env.requires.db.embedding.model` is not set or the ONNX model cannot be initialized +- A missing model in the project-local `.cds/models` cache is installed after a startup warning +- Starting `ai-sqlite` fails with a provisioning command if a configured model directory is missing or fails integrity checks +- Provisioning downloads are time-limited and accepted only when their expected size and SHA-256 match - Throws if embedding generation fails ## Test the plugin locally diff --git a/bin/cds-ai.js b/bin/cds-ai.js new file mode 100755 index 0000000..788d63f --- /dev/null +++ b/bin/cds-ai.js @@ -0,0 +1,10 @@ +#!/usr/bin/env node + +import { runModelCommand } from '../lib/vector_embedding/cli.js'; + +try { + await runModelCommand(process.argv.slice(2)); +} catch (error) { + process.stderr.write(`${error.message}\n`); + process.exitCode = 1; +} diff --git a/lib/sqlite/AISQLiteService.js b/lib/sqlite/AISQLiteService.js index cb5dc7e..498ba47 100644 --- a/lib/sqlite/AISQLiteService.js +++ b/lib/sqlite/AISQLiteService.js @@ -6,9 +6,30 @@ const LOG = cds.log('@cap-js/ai'); export default class AISQLiteService extends SQLiteService { async init() { - this._embeddingRuntime = await createEmbeddingRuntime(this.options.embedding); - LOG.info('Vector embedding ONNX model initialized'); - return super.init(); + this._embeddingRuntime = await createEmbeddingRuntime(this.options.embedding, { + root: cds.root, + warn: (message) => LOG.warn(message) + }); + try { + const service = await super.init(); + LOG.info('Vector embedding ONNX model initialized'); + return service; + } catch (error) { + await this._embeddingRuntime.dispose().catch(() => {}); + this._embeddingRuntime = undefined; + throw error; + } + } + + async disconnect(tenant) { + try { + return await super.disconnect(tenant); + } finally { + if (tenant === undefined) { + await this._embeddingRuntime?.dispose(); + this._embeddingRuntime = undefined; + } + } } get factory() { diff --git a/lib/vector_embedding/InferenceSession.js b/lib/vector_embedding/InferenceSession.js index 45ac693..ecb61fd 100644 --- a/lib/vector_embedding/InferenceSession.js +++ b/lib/vector_embedding/InferenceSession.js @@ -62,6 +62,13 @@ class InferenceSession { return output; } + dispose() { + const handler = this.handler; + if (!handler) return; + this.handler = undefined; + return handler.dispose(); + } + static async create(pathOrBuffer) { if (typeof pathOrBuffer !== 'string' && !(pathOrBuffer instanceof Uint8Array)) { throw new TypeError('Expected an ONNX model path or Uint8Array'); @@ -73,25 +80,34 @@ class InferenceSession { class SynchronousSessionHandler { constructor(pathOrBuffer) { this.session = new binding.InferenceSession(); - if (typeof pathOrBuffer === 'string') { - this.session.loadModel(pathOrBuffer, {}); - } else { - this.session.loadModel( - pathOrBuffer.buffer, - pathOrBuffer.byteOffset, - pathOrBuffer.byteLength, - {} - ); + try { + if (typeof pathOrBuffer === 'string') { + this.session.loadModel(pathOrBuffer, {}); + } else { + this.session.loadModel( + pathOrBuffer.buffer, + pathOrBuffer.byteOffset, + pathOrBuffer.byteLength, + {} + ); + } + this.inputNames = this.session.inputNames; + this.outputNames = this.session.outputNames; + } catch (error) { + try { + this.session.dispose(); + } catch { + // Preserve the model loading error. + } + throw error; } - this.inputNames = this.session.inputNames; - this.outputNames = this.session.outputNames; } run(feeds, fetches, options) { return this.session.run(feeds, fetches, options); } - async dispose() { + dispose() { this.session.dispose(); } } diff --git a/lib/vector_embedding/cli.js b/lib/vector_embedding/cli.js new file mode 100644 index 0000000..956c0ed --- /dev/null +++ b/lib/vector_embedding/cli.js @@ -0,0 +1,58 @@ +import { installModel } from './model-install.js'; +import { validateEmbeddingModel } from './embedding.js'; + +const HELP = `Usage: + npx @cap-js/ai install-model [--directory ] + +Options: + --directory Use this model-cache root instead of .cds/models + --help Show this help +`; + +async function runModelCommand(argv, options = {}) { + const { cwd = process.cwd(), stdout = process.stdout } = options; + const command = parseArguments(argv); + if (command.help) { + stdout.write(HELP); + return; + } + + const { modelDir } = await installModel(command.model, { + root: cwd, + directory: command.directory, + home: options.home, + fetchImpl: options.fetchImpl, + discover: options.discover, + validate: options.validate ?? validateEmbeddingModel, + timeoutMs: options.timeoutMs, + retryMs: options.retryMs + }); + stdout.write(`Installed ${command.model} in ${modelDir}\n`); +} + +function parseArguments(argv) { + if (argv.length === 0 || argv.includes('--help') || argv.includes('-h')) return { help: true }; + if (argv[0] !== 'install-model') { + throw new Error(`Unsupported command.\n\n${HELP}`); + } + + let model; + let directory; + for (let index = 1; index < argv.length; index++) { + const argument = argv[index]; + if (argument === '--directory') { + const value = argv[++index]; + if (!value || value.startsWith('--')) throw new Error(`${argument} requires a value`); + directory = value; + continue; + } + if (argument.startsWith('-')) throw new Error(`Unknown option '${argument}'`); + if (model) throw new Error(`Unexpected argument '${argument}'`); + model = argument; + } + + if (!model) throw new Error('Specify a model name'); + return { directory, model }; +} + +export { HELP, parseArguments, runModelCommand }; diff --git a/lib/vector_embedding/embedding.js b/lib/vector_embedding/embedding.js index 5c8c10f..63234b1 100644 --- a/lib/vector_embedding/embedding.js +++ b/lib/vector_embedding/embedding.js @@ -1,78 +1,169 @@ -import os from 'os'; -import path from 'path'; import { Tensor } from './InferenceSession.js'; +import { installModel } from './model-install.js'; import { - downloadModelIfNeeded, - getModelCacheDir, + getModelDirectory, + getModelRoot, loadModelAndTokenizer, - validateModelDescriptor + readModelLock, + verifyModelDirectory } from './model-utils.js'; const STANDARD_INPUT_NAMES = new Set(['input_ids', 'attention_mask', 'token_type_ids']); -const DEFAULT_MODEL = Object.freeze({ - repository: 'Xenova/all-MiniLM-L6-v2', - revision: '751bff37182d3f1213fa05d7196b954e230abad9', - dimensions: 384, - maxLength: 128, - files: Object.freeze([ - Object.freeze({ - role: 'model', - name: 'model.onnx', - path: 'onnx/model.onnx', - size: 90387606, - sha256: '759c3cd2b7fe7e93933ad23c4c9181b7396442a2ed746ec7c1d46192c469c46e' - }), - Object.freeze({ - role: 'tokenizer', - name: 'tokenizer.json', - path: 'tokenizer.json', - size: 711661, - sha256: 'da0e79933b9ed51798a3ae27893d3c5fa4a201126cef75586296df9b4d2c62a0' - }), - Object.freeze({ - role: 'tokenizerConfig', - name: 'tokenizer_config.json', - path: 'tokenizer_config.json', - size: 366, - sha256: '9261e7d79b44c8195c1cada2b453e55b00aeb81e907a6664974b4d7776172ab3' - }) - ]), - output: Object.freeze({ - name: 'last_hidden_state', - pooling: 'mean', - normalize: true - }) -}); - -async function createEmbeddingRuntime(model = DEFAULT_MODEL) { - validateModelDescriptor(model); - const modelDir = getModelCacheDir(getModelCacheRoot(), model); - await downloadModelIfNeeded(modelDir, model); +async function createEmbeddingRuntime(configuration, options = {}) { + const { model, modelDir } = await resolveEmbeddingModel(configuration, options); + return createEmbeddingRuntimeFromModel(modelDir, model); +} + +async function createEmbeddingRuntimeFromModel(modelDir, model) { const { session, tokenizer } = await loadModelAndTokenizer(modelDir, model); - const tokenizerState = createTokenizerState(tokenizer, model.maxLength); - - validateSession(session, model); - - const runtime = { - dimensions: model.dimensions, - embedding(text) { - const chunks = tokenizeWithChunks(String(text), tokenizer, tokenizerState); - return processChunkedEmbeddings(chunks, session, model); - }, - vectorEmbedding(text) { - if (!text) return JSON.stringify(new Array(model.dimensions).fill(0)); - return JSON.stringify(Array.from(this.embedding(text))); + let disposed = false; + const dispose = async () => { + if (disposed) return; + disposed = true; + await session.dispose(); + }; + + try { + const tokenizerState = createTokenizerState(tokenizer, model.maxLength); + + validateSession(session, model); + + const runtime = { + dimensions: model.dimensions, + embedding(text) { + const chunks = tokenizeWithChunks(String(text), tokenizer, tokenizerState); + return processChunkedEmbeddings(chunks, session, model); + }, + vectorEmbedding(text) { + if (!text) return JSON.stringify(new Array(model.dimensions).fill(0)); + return JSON.stringify(Array.from(this.embedding(text))); + }, + dispose + }; + + const probe = runtime.embedding('embedding model startup probe'); + if (probe.length !== model.dimensions) { + throw new Error( + `Embedding model produced ${probe.length} dimensions; configured ${model.dimensions}` + ); } + return runtime; + } catch (error) { + await dispose().catch(() => {}); + throw error; + } +} + +async function validateEmbeddingModel(modelDir, model) { + const runtime = await createEmbeddingRuntimeFromModel(modelDir, model); + await runtime.dispose(); +} + +async function resolveEmbeddingModel(configuration, options = {}) { + const { + root = process.cwd(), + warn = (message) => console.warn(message), + fetchImpl, + discover, + validate + } = options; + const { model: modelName, directory } = normalizeEmbeddingConfiguration(configuration); + const modelRoot = getModelRoot(directory, root, options.home); + const modelDir = getModelDirectory(modelRoot, modelName); + const installOptions = { + root, + directory: modelRoot, + home: options.home, + fetchImpl, + discover, + validate: validate ?? validateEmbeddingModel, + timeoutMs: options.provisionTimeoutMs, + retryMs: options.provisionRetryMs }; - const probe = runtime.embedding('embedding model startup probe'); - if (probe.length !== model.dimensions) { + let model; + try { + model = await readModelLock(modelDir); + } catch (error) { + if (directory !== undefined || !/Embedding model lock not found/.test(error.message)) { + const recovery = /Embedding model lock not found/.test(error.message) + ? modelInstallHint(modelName, directory) + : `Remove or replace the invalid lock explicitly, then ${lowercaseFirst( + modelInstallHint(modelName, directory) + )}`; + throw new Error(`${error.message}. ${recovery}`, { cause: error }); + } + return installModelOnDemand(modelName, modelDir, installOptions, warn); + } + + if (model.repository !== modelName) { + throw new Error( + `Embedding model directory ${modelDir} contains ${model.repository}, not ${modelName}. Choose another directory or provision the configured model there.` + ); + } + try { + await verifyModelDirectory(modelDir, model); + } catch (error) { + if (directory !== undefined) { + throw new Error(`${error.message}. ${modelInstallHint(modelName, directory)}`, { + cause: error + }); + } + return installModelOnDemand(modelName, modelDir, installOptions, warn); + } + return { model, modelDir }; +} + +async function installModelOnDemand(modelName, modelDir, options, warn) { + warn( + `Embedding model '${modelName}' is not available in '${modelDir}'. Downloading it now; application startup may be delayed. ${modelInstallHint(modelName)}` + ); + try { + return await installModel(modelName, options); + } catch (error) { + throw new Error( + `Failed to install embedding model '${modelName}': ${error.message}. ${modelInstallHint(modelName)}`, + { cause: error } + ); + } +} + +function normalizeEmbeddingConfiguration(configuration) { + if (configuration == null) { + throw new Error('cds.env.requires.db.embedding.model must be a non-empty string'); + } + if (typeof configuration !== 'object' || Array.isArray(configuration)) { + throw new TypeError('embedding must be an object with model and optional directory'); + } + + const unsupported = Object.keys(configuration).filter( + (name) => name !== 'model' && name !== 'directory' + ); + if (unsupported.length > 0) { throw new Error( - `Embedding model produced ${probe.length} dimensions; configured ${model.dimensions}` + `Unsupported embedding configuration: ${unsupported.join(', ')}. Only model and directory are supported.` ); } - return runtime; + if (typeof configuration.model !== 'string' || !configuration.model.trim()) { + throw new Error('cds.env.requires.db.embedding.model must be a non-empty string'); + } + if ( + configuration.directory !== undefined && + (typeof configuration.directory !== 'string' || !configuration.directory.trim()) + ) { + throw new Error('embedding.directory must be a non-empty string'); + } + return { model: configuration.model, directory: configuration.directory }; +} + +function modelInstallHint(repository, directory) { + const directoryArgument = directory === undefined ? '' : ` --directory ${directory}`; + return `Run 'npx @cap-js/ai install-model ${repository}${directoryArgument}'.`; +} + +function lowercaseFirst(value) { + return `${value[0].toLowerCase()}${value.slice(1)}`; } function createTokenizerState(tokenizer, maxLength) { @@ -287,42 +378,28 @@ function validateAttentionMask(mask) { return mask; } -function getDataDir(appName = 'semantic-search') { - const home = os.homedir(); - const directory = - os.platform() === 'win32' - ? process.env.LOCALAPPDATA || process.env.APPDATA || path.join(home, 'AppData', 'Local') - : process.env.XDG_DATA_HOME || path.join(home, '.local', 'share'); - - return path.join(directory, appName); -} - -function getModelCacheRoot() { - return process.env.CDS_AI_MODEL_CACHE || path.join(getDataDir(), 'models'); -} - -let defaultRuntime; -let defaultInitialization; +let sessionRuntime; +let sessionInitialization; -async function createSession() { - defaultInitialization ??= createEmbeddingRuntime() - .then((runtime) => (defaultRuntime = runtime)) +async function createSession(configuration, options) { + sessionInitialization ??= createEmbeddingRuntime(configuration, options) + .then((runtime) => (sessionRuntime = runtime)) .catch((error) => { - defaultInitialization = undefined; + sessionInitialization = undefined; throw error; }); - return defaultInitialization; + return sessionInitialization; } function embedding(text) { - if (!defaultRuntime) { + if (!sessionRuntime) { throw new Error( 'Embedding session not initialized. Call createSession() before using embedding().' ); } const chunk = { content: text }; return Object.defineProperty(chunk, 'embedding', { - value: defaultRuntime.embedding(text), + value: sessionRuntime.embedding(text), writable: true, configurable: true, enumerable: false @@ -330,14 +407,16 @@ function embedding(text) { } export { - DEFAULT_MODEL, createSession, createEmbeddingRuntime, + createEmbeddingRuntimeFromModel, createFeeds, createTokenizerState, embedding, poolOutput, - tokenizeWithChunks + resolveEmbeddingModel, + tokenizeWithChunks, + validateEmbeddingModel }; export default embedding; diff --git a/lib/vector_embedding/index.js b/lib/vector_embedding/index.js index 4089a45..aad5cca 100644 --- a/lib/vector_embedding/index.js +++ b/lib/vector_embedding/index.js @@ -3,14 +3,11 @@ import * as embeddingModule from './embedding.js'; const LOG = cds.log('@cap-js/ai'); let loggedInitialization = false; -const { dimensions: DEFAULT_DIMENSIONS } = embeddingModule.DEFAULT_MODEL; -const modelDimensions = { - 'SAP_GXY.20250407': DEFAULT_DIMENSIONS, - 'SAP_GXY.20240715': DEFAULT_DIMENSIONS -}; +let dimensions; -async function initializeEmbedding() { - await embeddingModule.createSession(); +async function initializeEmbedding(configuration, options) { + const runtime = await embeddingModule.createSession(configuration, options); + dimensions = runtime.dimensions; if (!loggedInitialization) { LOG.info('Vector embedding ONNX model initialized'); loggedInitialization = true; @@ -20,10 +17,14 @@ async function initializeEmbedding() { function vector_embedding(text, text_type, model_and_version) { void text_type; // Retained for HANA-compatible function arity. + void model_and_version; // The configured embedding model determines vector dimensions. if (text) return JSON.stringify(Array.from(embeddingModule.embedding(text).embedding)); - return JSON.stringify( - new Array(modelDimensions[model_and_version] ?? DEFAULT_DIMENSIONS).fill(0) - ); + if (!dimensions) { + throw new Error( + 'Embedding session not initialized. Call initializeEmbedding() with an embedding configuration before using vector_embedding().' + ); + } + return JSON.stringify(new Array(dimensions).fill(0)); } export { initializeEmbedding, vector_embedding }; diff --git a/lib/vector_embedding/model-discovery.js b/lib/vector_embedding/model-discovery.js new file mode 100644 index 0000000..b2a716f --- /dev/null +++ b/lib/vector_embedding/model-discovery.js @@ -0,0 +1,390 @@ +import { createHash } from 'crypto'; + +import { assertSafeRepository, validateModelDescriptor } from './model-utils.js'; + +const HUGGING_FACE_ORIGIN = 'https://huggingface.co'; +const REQUIRED_ARTIFACTS = [ + { role: 'model', name: 'model.onnx', path: 'onnx/model.onnx' }, + { role: 'tokenizer', name: 'tokenizer.json', path: 'tokenizer.json' }, + { + role: 'tokenizerConfig', + name: 'tokenizer_config.json', + path: 'tokenizer_config.json' + }, + { role: 'auxiliary', name: 'config.json', path: 'config.json' } +]; +const MODULES_FILE = 'modules.json'; +const SENTENCE_CONFIG_FILE = 'sentence_bert_config.json'; +const SUPPORTED_MODULES = new Set([ + 'sentence_transformers.models.Transformer', + 'sentence_transformers.models.Pooling', + 'sentence_transformers.models.Normalize' +]); + +async function discoverModel(repository, options = {}) { + assertSafeRepository(repository); + const fetchImpl = + typeof options === 'function' ? options : (options.fetchImpl ?? globalThis.fetch); + if (typeof fetchImpl !== 'function') throw new TypeError('A fetch implementation is required'); + + const context = createContext( + fetchImpl, + typeof options === 'function' ? undefined : options.origin + ); + const modelInfo = await fetchModelInfo(context, repository); + const revision = immutableRevision(modelInfo, repository); + const siblings = siblingMap(modelInfo, repository); + + const tokenizer = await fetchJsonFile(context, repository, revision, 'tokenizer.json'); + const tokenizerConfig = await fetchJsonFile( + context, + repository, + revision, + 'tokenizer_config.json' + ); + const config = await fetchJsonFile(context, repository, revision, 'config.json'); + const dimensions = positiveInteger(config.value.hidden_size); + if (!dimensions) { + throw new Error(`Cannot determine embedding dimensions from '${repository}/config.json'`); + } + + const selected = REQUIRED_ARTIFACTS.map((artifact) => { + const sibling = siblings.get(artifact.path); + if (!sibling) { + throw new Error( + `Hugging Face model '${repository}' must contain the exact file '${artifact.path}'` + ); + } + return { ...artifact, sibling }; + }); + const externalData = [...siblings.values()] + .filter(({ rfilename }) => /^onnx\/model\.onnx_data(?:$|[._-])/u.test(rfilename)) + .map((sibling) => ({ + role: 'auxiliary', + name: sibling.rfilename.slice('onnx/'.length), + path: sibling.rfilename, + sibling + })); + if (usesExternalData(config.value) && externalData.length === 0) { + throw new Error( + `Hugging Face model '${repository}' declares external ONNX data but does not contain 'onnx/model.onnx_data'` + ); + } + selected.push(...externalData); + + const semantics = await discoverSentenceTransformerSemantics( + context, + repository, + modelInfo, + new Set() + ); + const maxLength = minimumPositiveInteger([ + tokenizer.value?.truncation?.max_length, + tokenizerConfig.value.max_length, + semantics.maxLength, + tokenizerConfig.value.model_max_length, + config.value.max_position_embeddings + ]); + if (!maxLength) { + throw new Error(`Cannot determine the maximum input length for '${repository}'`); + } + + const knownFiles = new Map([ + ['tokenizer.json', tokenizer], + ['tokenizer_config.json', tokenizerConfig], + ['config.json', config] + ]); + const files = await Promise.all( + selected.map(async ({ sibling, ...artifact }) => ({ + ...artifact, + ...(await discoverFileIntegrity( + context, + repository, + revision, + sibling, + knownFiles.get(artifact.path) + )) + })) + ); + + return validateModelDescriptor({ + repository, + revision, + dimensions, + maxLength, + files, + output: { + name: 'last_hidden_state', + pooling: semantics.pooling, + normalize: semantics.normalize + } + }); +} + +function createContext(fetchImpl, origin = HUGGING_FACE_ORIGIN) { + if (typeof origin !== 'string' || !origin.trim()) { + throw new TypeError('The Hugging Face origin must be a non-empty string'); + } + return { fetchImpl, origin: origin.replace(/\/$/, ''), jsonFiles: new Map() }; +} + +function usesExternalData(config) { + const value = config?.['transformers.js_config']?.use_external_data_format; + return value === true || value?.['model.onnx'] === 1 || value?.['model.onnx'] === true; +} + +async function fetchModelInfo(context, repository) { + const url = `${context.origin}/api/models/${repositoryPath(repository)}?blobs=true`; + const response = await checkedFetch(context, url); + const value = await readJsonResponse(response, url); + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`Invalid Hugging Face model information for '${repository}'`); + } + return value; +} + +function immutableRevision(modelInfo, repository) { + if (typeof modelInfo.sha !== 'string' || !/^[a-fA-F0-9]{40,64}$/.test(modelInfo.sha)) { + throw new Error(`Hugging Face did not return an immutable revision for '${repository}'`); + } + return modelInfo.sha.toLowerCase(); +} + +function siblingMap(modelInfo, repository) { + if (!Array.isArray(modelInfo.siblings)) { + throw new Error(`Hugging Face did not return a file list for '${repository}'`); + } + const siblings = new Map(); + for (const sibling of modelInfo.siblings) { + if (typeof sibling?.rfilename !== 'string') continue; + if (siblings.has(sibling.rfilename)) { + throw new Error(`Hugging Face returned duplicate file '${sibling.rfilename}'`); + } + siblings.set(sibling.rfilename, sibling); + } + return siblings; +} + +async function discoverSentenceTransformerSemantics(context, repository, modelInfo, visited) { + if (visited.has(repository)) { + throw new Error(`Circular Hugging Face base_model chain involving '${repository}'`); + } + visited.add(repository); + + const revision = immutableRevision(modelInfo, repository); + const siblings = siblingMap(modelInfo, repository); + if (siblings.has(MODULES_FILE)) { + return readSentenceTransformerSemantics(context, repository, revision, siblings); + } + + const baseModel = baseModelRepository(modelInfo.cardData?.base_model); + if (!baseModel) { + throw new Error( + `Cannot determine pooling and normalization for '${repository}': no Sentence Transformers modules or unambiguous base_model metadata` + ); + } + assertSafeRepository(baseModel); + const baseInfo = await fetchModelInfo(context, baseModel); + return discoverSentenceTransformerSemantics(context, baseModel, baseInfo, visited); +} + +async function readSentenceTransformerSemantics(context, repository, revision, siblings) { + const modules = (await fetchJsonFile(context, repository, revision, MODULES_FILE)).value; + if (!Array.isArray(modules)) { + throw new Error(`Invalid Sentence Transformers modules in '${repository}/${MODULES_FILE}'`); + } + + for (const module of modules) { + if (!module || typeof module.type !== 'string' || !SUPPORTED_MODULES.has(module.type)) { + throw new Error( + `Unsupported Sentence Transformers module '${module?.type ?? 'unknown'}' in '${repository}'` + ); + } + } + const expectedTypes = [ + 'sentence_transformers.models.Transformer', + 'sentence_transformers.models.Pooling' + ]; + if (modules.length === 3) expectedTypes.push('sentence_transformers.models.Normalize'); + if ( + modules.length < 2 || + modules.length > 3 || + modules.some((module, index) => module.type !== expectedTypes[index]) + ) { + throw new Error(`Cannot determine an unambiguous pooling pipeline for '${repository}'`); + } + + const poolingModule = modules[1]; + const poolingPath = moduleConfigPath(poolingModule, repository); + if (!siblings.has(poolingPath)) { + throw new Error(`Sentence Transformers pooling configuration '${poolingPath}' is missing`); + } + const poolingConfig = (await fetchJsonFile(context, repository, revision, poolingPath)).value; + const pooling = determinePooling(poolingConfig, repository); + + let maxLength; + const transformer = modules[0]; + const sentenceConfigPaths = [SENTENCE_CONFIG_FILE]; + if (transformer?.path) { + sentenceConfigPaths.unshift( + `${normalizedModulePath(transformer.path)}/${SENTENCE_CONFIG_FILE}` + ); + } + const sentenceConfigPath = sentenceConfigPaths.find((configPath) => siblings.has(configPath)); + if (sentenceConfigPath) { + const sentenceConfig = await fetchJsonFile(context, repository, revision, sentenceConfigPath); + maxLength = positiveInteger(sentenceConfig.value.max_seq_length); + } + + return { pooling, normalize: modules.length === 3, maxLength }; +} + +function moduleConfigPath(module, repository) { + const modulePath = normalizedModulePath(module.path); + if (!modulePath) { + throw new Error(`Sentence Transformers pooling module in '${repository}' has no path`); + } + return `${modulePath}/config.json`; +} + +function normalizedModulePath(value) { + if ( + typeof value !== 'string' || + !value || + value.includes('\\') || + value.startsWith('/') || + value.split('/').some((part) => !part || part === '.' || part === '..') + ) { + return undefined; + } + return value; +} + +function determinePooling(config, repository) { + if (!config || typeof config !== 'object' || Array.isArray(config)) { + throw new Error(`Invalid Sentence Transformers pooling configuration for '${repository}'`); + } + const enabled = [ + ['cls', config.pooling_mode_cls_token], + ['mean', config.pooling_mode_mean_tokens], + ['max', config.pooling_mode_max_tokens], + ['mean_sqrt_len', config.pooling_mode_mean_sqrt_len_tokens], + ['weightedmean', config.pooling_mode_weightedmean_tokens], + ['lasttoken', config.pooling_mode_lasttoken] + ].filter(([, value]) => value === true); + if (config.pooling_mode !== undefined) { + if (!['mean', 'cls'].includes(config.pooling_mode)) { + throw new Error(`Unsupported or ambiguous Sentence Transformers pooling for '${repository}'`); + } + if (enabled.length > 0 && (enabled.length !== 1 || enabled[0][0] !== config.pooling_mode)) { + throw new Error(`Unsupported or ambiguous Sentence Transformers pooling for '${repository}'`); + } + return config.pooling_mode; + } + if (enabled.length !== 1 || !['mean', 'cls'].includes(enabled[0][0])) { + throw new Error(`Unsupported or ambiguous Sentence Transformers pooling for '${repository}'`); + } + return enabled[0][0]; +} + +function baseModelRepository(value) { + if (typeof value === 'string') return value; + if (Array.isArray(value) && value.length === 1 && typeof value[0] === 'string') return value[0]; + if (value && typeof value === 'object' && !Array.isArray(value) && typeof value.id === 'string') { + return value.id; + } + return undefined; +} + +async function discoverFileIntegrity(context, repository, revision, sibling, knownFile) { + const metadataChecksum = lfsChecksum(sibling); + const metadataSize = positiveInteger(sibling.size) ?? positiveInteger(sibling.lfs?.size); + if (metadataChecksum && metadataSize) { + return { size: metadataSize, sha256: metadataChecksum }; + } + + const file = knownFile ?? (await fetchFile(context, repository, revision, sibling.rfilename)); + return { + size: file.bytes.byteLength, + sha256: createHash('sha256').update(file.bytes).digest('hex') + }; +} + +function lfsChecksum(sibling) { + const candidate = sibling.lfs?.sha256 ?? sibling.lfs?.oid; + if (typeof candidate !== 'string') return undefined; + const checksum = candidate.replace(/^sha256:/, '').toLowerCase(); + return /^[a-f0-9]{64}$/.test(checksum) ? checksum : undefined; +} + +async function fetchJsonFile(context, repository, revision, remotePath) { + const key = `${repository}@${revision}/${remotePath}`; + let file = context.jsonFiles.get(key); + if (!file) { + file = fetchFile(context, repository, revision, remotePath).then(({ bytes, url }) => { + try { + return { bytes, value: JSON.parse(bytes.toString('utf8')) }; + } catch (error) { + throw new Error(`Invalid JSON returned from ${url}: ${error.message}`, { cause: error }); + } + }); + context.jsonFiles.set(key, file); + } + return file; +} + +async function fetchFile(context, repository, revision, remotePath) { + const url = `${context.origin}/${repositoryPath(repository)}/resolve/${revision}/${remotePath + .split('/') + .map(encodeURIComponent) + .join('/')}`; + const response = await checkedFetch(context, url); + return { bytes: await readBytes(response), url }; +} + +async function checkedFetch(context, url) { + let response; + try { + response = await context.fetchImpl(url); + } catch (error) { + throw new Error(`Cannot fetch ${url}: ${error.message}`, { cause: error }); + } + if (!response || response.ok !== true) { + throw new Error(`Cannot fetch ${url}: HTTP ${response?.status ?? 'unknown'}`); + } + return response; +} + +async function readJsonResponse(response, url) { + try { + if (typeof response.json === 'function') return await response.json(); + return JSON.parse((await readBytes(response)).toString('utf8')); + } catch (error) { + throw new Error(`Invalid JSON returned from ${url}: ${error.message}`, { cause: error }); + } +} + +async function readBytes(response) { + if (typeof response.arrayBuffer === 'function') { + return Buffer.from(await response.arrayBuffer()); + } + if (typeof response.text === 'function') return Buffer.from(await response.text()); + throw new Error('Fetch response does not expose arrayBuffer() or text()'); +} + +function positiveInteger(value) { + return Number.isSafeInteger(value) && value > 0 ? value : undefined; +} + +function minimumPositiveInteger(values) { + const candidates = values.map(positiveInteger).filter((value) => value !== undefined); + return candidates.length > 0 ? Math.min(...candidates) : undefined; +} + +function repositoryPath(repository) { + return repository.split('/').map(encodeURIComponent).join('/'); +} + +const discoverModelDescriptor = discoverModel; + +export { discoverModel, discoverModelDescriptor }; diff --git a/lib/vector_embedding/model-install.js b/lib/vector_embedding/model-install.js new file mode 100644 index 0000000..13f2ed3 --- /dev/null +++ b/lib/vector_embedding/model-install.js @@ -0,0 +1,64 @@ +import { setTimeout as delay } from 'node:timers/promises'; +import { discoverModel } from './model-discovery.js'; +import { + MODEL_PROVISIONING_IN_PROGRESS, + assertSafeRepository, + getModelDirectory, + getModelRoot, + provisionModel, + readModelLock +} from './model-utils.js'; + +const MODEL_PROVISION_TIMEOUT_MS = 15 * 60 * 1000; +const MODEL_PROVISION_RETRY_MS = 250; + +async function installModel(repository, options = {}) { + assertSafeRepository(repository); + const modelRoot = getModelRoot(options.directory, options.root, options.home); + const modelDir = getModelDirectory(modelRoot, repository); + const discover = options.discover ?? discoverModel; + + let model; + try { + model = await readModelLock(modelDir); + assertRepository(model, repository, modelDir); + } catch (error) { + if (!/Embedding model lock not found/.test(error.message)) throw error; + model = await discover(repository, { fetchImpl: options.fetchImpl }); + assertRepository(model, repository, modelDir); + } + + const deadline = Date.now() + (options.timeoutMs ?? MODEL_PROVISION_TIMEOUT_MS); + const retryMs = options.retryMs ?? MODEL_PROVISION_RETRY_MS; + while (true) { + try { + // eslint-disable-next-line no-await-in-loop + await provisionModel(model, { + directory: modelDir, + fetchImpl: options.fetchImpl, + validate: options.validate + }); + return { model, modelDir, modelRoot }; + } catch (error) { + if (error.code !== MODEL_PROVISIONING_IN_PROGRESS) throw error; + const remaining = deadline - Date.now(); + if (remaining <= 0) { + throw new Error(`Timed out waiting for embedding model provisioning in ${modelDir}`, { + cause: error + }); + } + // eslint-disable-next-line no-await-in-loop + await delay(Math.min(retryMs, remaining)); + } + } +} + +function assertRepository(model, repository, modelDir) { + if (model.repository !== repository) { + throw new Error( + `Embedding model directory ${modelDir} contains ${model.repository}, not ${repository}. Choose another directory or remove it explicitly before installing the configured model.` + ); + } +} + +export { installModel }; diff --git a/lib/vector_embedding/model-utils.js b/lib/vector_embedding/model-utils.js index d3be040..5fe5728 100644 --- a/lib/vector_embedding/model-utils.js +++ b/lib/vector_embedding/model-utils.js @@ -1,13 +1,20 @@ import { createHash, randomUUID } from 'crypto'; import { createReadStream } from 'fs'; import fs from 'fs/promises'; +import os from 'os'; import path from 'path'; -import { Tokenizer } from '@huggingface/tokenizers'; -import { InferenceSession } from './InferenceSession.js'; const DOWNLOAD_TIMEOUT_MS = 5 * 60 * 1000; +const MODEL_LOCK_FILE = 'embedding.lock.json'; +const MODEL_INSTALL_LOCK_FILE = '.embedding.install.lock'; +const MODEL_LOCK_VERSION = 1; +const MODEL_PROVISIONING_IN_PROGRESS = 'ERR_EMBEDDING_MODEL_PROVISIONING_IN_PROGRESS'; +const INSTALL_LOCK_STALE_MS = 30 * 60 * 1000; +const PROVISIONED_DIRECTORY_MODE = 0o755; +const PROVISIONED_FILE_MODE = 0o644; const REQUIRED_FILE_ROLES = ['model', 'tokenizer', 'tokenizerConfig']; const ALLOWED_FILE_ROLES = new Set([...REQUIRED_FILE_ROLES, 'auxiliary']); +const RESERVED_ARTIFACT_PATHS = [MODEL_LOCK_FILE, MODEL_INSTALL_LOCK_FILE]; function validateModelDescriptor(model) { if (!model || typeof model !== 'object' || Array.isArray(model)) { @@ -44,6 +51,10 @@ function validateModelDescriptor(model) { assertSafeRelativePath(file.name, `embedding.files[${file.role}].name`); assertSafeRelativePath(file.path, `embedding.files[${file.role}].path`); if (names.has(file.name)) throw new Error(`Duplicate embedding file name '${file.name}'`); + for (const existingName of names) assertNoPathCollision(file.name, existingName); + for (const reservedPath of RESERVED_ARTIFACT_PATHS) { + assertNoPathCollision(file.name, reservedPath, 'provisioning metadata'); + } names.add(file.name); if (!Number.isSafeInteger(file.size) || file.size < 1) { throw new Error(`Invalid size for embedding file '${file.name}'`); @@ -73,6 +84,18 @@ function validateModelDescriptor(model) { return model; } +function assertNoPathCollision(left, right, description = 'another embedding file') { + const normalizedLeft = left.toLowerCase(); + const normalizedRight = right.toLowerCase(); + if ( + normalizedLeft === normalizedRight || + normalizedLeft.startsWith(`${normalizedRight}/`) || + normalizedRight.startsWith(`${normalizedLeft}/`) + ) { + throw new Error(`Embedding file '${left}' conflicts with ${description} '${right}'`); + } +} + function assertSafeRepository(repository) { if ( typeof repository !== 'string' || @@ -83,25 +106,45 @@ function assertSafeRepository(repository) { } } +function getModelRoot(directory, root = process.cwd(), home = os.homedir()) { + if (directory === undefined) return path.join(root, '.cds', 'models'); + if (directory === '~') return home; + if (/^~[\\/]/.test(directory)) return path.join(home, directory.slice(2)); + return path.resolve(root, directory); +} + +function getModelDirectory(root, repository) { + assertSafeRepository(repository); + return path.join(root, ...repository.split('/')); +} + function assertSafeRelativePath(value, field) { + const parts = typeof value === 'string' ? value.split('/') : []; if ( typeof value !== 'string' || value.length === 0 || value.includes('\\') || path.posix.isAbsolute(value) || path.posix.normalize(value) !== value || - value.split('/').some((part) => part === '' || part === '.' || part === '..') || + parts.some((part) => part === '' || part === '.' || part === '..') || + parts.some((part) => part.endsWith('.') || isWindowsDeviceName(part)) || !/^[A-Za-z0-9._/-]+$/.test(value) ) { throw new Error(`${field} must be a safe relative path`); } } +function isWindowsDeviceName(value) { + const basename = value.split('.')[0].toUpperCase(); + return /^(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])$/.test(basename); +} + function fileForRole(model, role) { return model.files.find((file) => file.role === role); } -function artifactSetDigest(model) { +function modelDescriptorDigest(model) { + validateModelDescriptor(model); const files = model.files .map(({ role, name, path: remotePath, size, sha256: checksum }) => ({ role, @@ -114,21 +157,18 @@ function artifactSetDigest(model) { const canonical = JSON.stringify({ repository: model.repository, revision: model.revision.toLowerCase(), - files + dimensions: model.dimensions, + maxLength: model.maxLength, + files, + output: { + name: model.output.name, + pooling: model.output.pooling, + normalize: model.output.normalize + } }); return createHash('sha256').update(canonical).digest('hex'); } -function getModelCacheDir(cacheRoot, model) { - validateModelDescriptor(model); - return path.join( - cacheRoot, - ...model.repository.split('/'), - model.revision.toLowerCase(), - artifactSetDigest(model) - ); -} - async function sha256(filePath) { const hash = createHash('sha256'); for await (const chunk of createReadStream(filePath)) hash.update(chunk); @@ -137,7 +177,7 @@ async function sha256(filePath) { async function isValidFile(filePath, file) { try { - const stat = await fs.stat(filePath); + const stat = await fs.lstat(filePath); return stat.isFile() && stat.size === file.size && (await sha256(filePath)) === file.sha256; } catch (error) { if (error.code === 'ENOENT') return false; @@ -145,6 +185,27 @@ async function isValidFile(filePath, file) { } } +async function verifyModelDirectory(modelDir, model) { + validateModelDescriptor(model); + await assertModelDirectory(modelDir); + const validity = await Promise.all( + model.files.map(async (file) => { + await assertNoSymlinkComponents(modelDir, file.name); + return { + file, + valid: await isValidFile(path.join(modelDir, file.name), file) + }; + }) + ); + const invalid = validity.filter(({ valid }) => !valid).map(({ file }) => file.name); + if (invalid.length > 0) { + throw new Error( + `Embedding model is not provisioned or failed integrity checks in ${modelDir}. Missing or invalid files: ${invalid.join(', ')}` + ); + } + return modelDir; +} + async function downloadFile(url, outputPath, file, options = {}) { const { fetchImpl = globalThis.fetch, timeoutMs = DOWNLOAD_TIMEOUT_MS } = options; const controller = new AbortController(); @@ -167,7 +228,7 @@ async function downloadFile(url, outputPath, file, options = {}) { } await fs.mkdir(path.dirname(outputPath), { recursive: true }); - handle = await fs.open(temporaryPath, 'wx', 0o600); + handle = await fs.open(temporaryPath, 'wx', PROVISIONED_FILE_MODE); const hash = createHash('sha256'); let bytesWritten = 0; @@ -193,6 +254,8 @@ async function downloadFile(url, outputPath, file, options = {}) { throw new Error(`Invalid SHA-256 for ${url}: expected ${file.sha256}, received ${digest}`); } + await fs.chmod(temporaryPath, PROVISIONED_FILE_MODE); + try { await fs.rename(temporaryPath, outputPath); } catch (error) { @@ -203,6 +266,7 @@ async function downloadFile(url, outputPath, file, options = {}) { await fs.rename(temporaryPath, outputPath); } } + await fs.chmod(outputPath, PROVISIONED_FILE_MODE); } catch (error) { if (error.name === 'AbortError') { throw new Error(`Timed out after ${timeoutMs} ms while downloading ${url}`, { cause: error }); @@ -217,12 +281,19 @@ async function downloadFile(url, outputPath, file, options = {}) { async function downloadModelIfNeeded(modelDir, model, options) { validateModelDescriptor(model); - await fs.mkdir(modelDir, { recursive: true }); + await ensureDirectory(modelDir); for (const file of model.files) { const filePath = path.join(modelDir, file.name); // eslint-disable-next-line no-await-in-loop - if (await isValidFile(filePath, file)) continue; + await prepareArtifactPath(modelDir, file.name); + // eslint-disable-next-line no-await-in-loop + if (await isValidFile(filePath, file)) { + // Keep build-time provisioning readable when the runtime uses another UID. + // eslint-disable-next-line no-await-in-loop + await fs.chmod(filePath, PROVISIONED_FILE_MODE); + continue; + } const url = `https://huggingface.co/${model.repository}/resolve/${model.revision}/${file.path}`; // Files are downloaded serially to avoid multiplying startup bandwidth and memory usage. @@ -231,7 +302,418 @@ async function downloadModelIfNeeded(modelDir, model, options) { } } +async function prepareArtifactPath(modelDir, relativePath) { + await assertNoSymlinkComponents(modelDir, relativePath); + let current = modelDir; + for (const part of relativePath.split('/').slice(0, -1)) { + current = path.join(current, part); + // eslint-disable-next-line no-await-in-loop + await fs.mkdir(current, { mode: PROVISIONED_DIRECTORY_MODE }).catch((error) => { + if (error.code !== 'EEXIST') throw error; + }); + // eslint-disable-next-line no-await-in-loop + const stat = await fs.lstat(current); + if (stat.isSymbolicLink()) { + throw new Error(`Embedding artifact path must not contain symbolic links: ${current}`); + } + if (!stat.isDirectory()) { + throw new Error(`Embedding artifact parent is not a directory: ${current}`); + } + // eslint-disable-next-line no-await-in-loop + await fs.chmod(current, PROVISIONED_DIRECTORY_MODE); + } + await assertNoSymlinkComponents(modelDir, relativePath); +} + +async function assertNoSymlinkComponents(modelDir, relativePath) { + const exists = await assertModelDirectory(modelDir); + if (!exists) return; + let current = modelDir; + const parts = relativePath.split('/'); + for (let index = 0; index < parts.length; index++) { + current = path.join(current, parts[index]); + let stat; + try { + // eslint-disable-next-line no-await-in-loop + stat = await fs.lstat(current); + } catch (error) { + if (error.code === 'ENOENT') return; + throw error; + } + if (stat.isSymbolicLink()) { + throw new Error(`Embedding artifact path must not contain symbolic links: ${current}`); + } + if (index < parts.length - 1 && !stat.isDirectory()) { + throw new Error(`Embedding artifact parent is not a directory: ${current}`); + } + } +} + +async function assertModelDirectory(modelDir) { + try { + const stat = await fs.lstat(modelDir); + if (stat.isSymbolicLink()) { + throw new Error(`Embedding model directory must not be a symbolic link: ${modelDir}`); + } + if (!stat.isDirectory()) { + throw new Error(`Embedding model path is not a directory: ${modelDir}`); + } + return true; + } catch (error) { + if (error.code === 'ENOENT') return false; + throw error; + } +} + +async function ensureDirectory(directory) { + await ensureParentDirectories(directory); + await assertModelDirectory(directory); + await fs.chmod(directory, PROVISIONED_DIRECTORY_MODE); +} + +async function ensureParentDirectories(directory) { + const missing = []; + let current = directory; + // eslint-disable-next-line no-await-in-loop + while (!(await pathExists(current))) { + missing.push(current); + const parent = path.dirname(current); + if (parent === current) break; + current = parent; + } + + for (const missingDirectory of missing.reverse()) { + // eslint-disable-next-line no-await-in-loop + await fs.mkdir(missingDirectory, { mode: PROVISIONED_DIRECTORY_MODE }).catch((error) => { + if (error.code !== 'EEXIST') throw error; + }); + // Explicit chmod avoids umask making build-time model directories unreadable at runtime. + // eslint-disable-next-line no-await-in-loop + await fs.chmod(missingDirectory, PROVISIONED_DIRECTORY_MODE); + } +} + +async function pathExists(filePath) { + try { + await fs.lstat(filePath); + return true; + } catch (error) { + if (error.code === 'ENOENT') return false; + throw error; + } +} + +async function canonicalizeProvisioningDirectory(directory) { + const requestedDirectory = path.resolve(directory); + let current = requestedDirectory; + const missing = []; + + while (true) { + try { + // eslint-disable-next-line no-await-in-loop + const stat = await fs.lstat(current); + if (current === requestedDirectory && stat.isSymbolicLink()) { + throw new Error(`Embedding model directory must not be a symbolic link: ${directory}`); + } + // eslint-disable-next-line no-await-in-loop + const canonicalAncestor = await fs.realpath(current); + // eslint-disable-next-line no-await-in-loop + const canonicalStat = await fs.stat(canonicalAncestor); + if (!canonicalStat.isDirectory()) { + throw new Error(`Embedding model path is not a directory: ${current}`); + } + return path.join(canonicalAncestor, ...missing.reverse()); + } catch (error) { + if (error.code !== 'ENOENT') throw error; + missing.push(path.basename(current)); + const parent = path.dirname(current); + if (parent === current) throw error; + current = parent; + } + } +} + +async function readModelLock(modelDir) { + const lockPath = path.join(modelDir, MODEL_LOCK_FILE); + let lock; + try { + await assertModelDirectory(modelDir); + const stat = await fs.lstat(lockPath); + if (stat.isSymbolicLink()) { + throw new Error(`Embedding model lock must not be a symbolic link at ${lockPath}`); + } + lock = JSON.parse(await fs.readFile(lockPath, 'utf8')); + } catch (error) { + if (error.code === 'ENOENT') { + throw new Error(`Embedding model lock not found at ${lockPath}`, { cause: error }); + } + throw new Error(`Cannot read embedding model lock at ${lockPath}: ${error.message}`, { + cause: error + }); + } + if (lock.formatVersion !== MODEL_LOCK_VERSION) { + throw new Error( + `Unsupported embedding model lock version ${lock.formatVersion ?? 'missing'} at ${lockPath}` + ); + } + const { formatVersion, ...model } = lock; + void formatVersion; + return validateModelDescriptor(model); +} + +async function writeModelLock(modelDir, model) { + const lockPath = path.join(modelDir, MODEL_LOCK_FILE); + const temporaryPath = `${lockPath}.${process.pid}.${randomUUID()}.tmp`; + const contents = `${JSON.stringify({ ...model, formatVersion: MODEL_LOCK_VERSION }, null, 2)}\n`; + let handle; + try { + await ensureDirectory(modelDir); + handle = await fs.open(temporaryPath, 'wx', PROVISIONED_FILE_MODE); + await handle.writeFile(contents); + await handle.sync(); + await handle.close(); + handle = undefined; + await fs.chmod(temporaryPath, PROVISIONED_FILE_MODE); + await fs.rename(temporaryPath, lockPath); + await fs.chmod(lockPath, PROVISIONED_FILE_MODE); + } finally { + await handle?.close().catch(() => {}); + await fs.unlink(temporaryPath).catch(() => {}); + } +} + +async function provisionModel(model, options = {}) { + validateModelDescriptor(model); + if (typeof options.directory !== 'string' || !options.directory.trim()) { + throw new Error('A non-empty provisioning directory is required'); + } + const requestedDirectory = path.resolve(options.directory); + const directory = await canonicalizeProvisioningDirectory(requestedDirectory); + await ensureParentDirectories(path.dirname(directory)); + + return withInstallLock(directory, async () => { + const directoryExists = await assertModelDirectory(directory); + let lockedModel; + try { + lockedModel = await readModelLock(directory); + if (modelDescriptorDigest(lockedModel) !== modelDescriptorDigest(model)) { + throw new Error( + `Embedding model directory ${directory} is locked to a different model descriptor for ${lockedModel.repository}@${lockedModel.revision}. Choose another directory or remove it explicitly.` + ); + } + } catch (error) { + if (!/Embedding model lock not found/.test(error.message)) throw error; + } + + if (directoryExists) { + try { + await verifyModelDirectory(directory, model); + if (!lockedModel) await writeModelLock(directory, model); + await makeModelDirectoryReadable(directory, model); + await options.validate?.(directory, model); + return directory; + } catch (error) { + if (/symbolic link/.test(error.message)) throw error; + if (!lockedModel && (await directoryHasEntries(directory))) { + throw new Error( + `Embedding model directory ${directory} is not empty and has no valid lock. Choose an empty directory or remove its contents explicitly.`, + { cause: error } + ); + } + } + } + + const stagingDirectory = await createStagingDirectory(directory); + let published = false; + try { + await downloadModelIfNeeded(stagingDirectory, model, options); + await verifyModelDirectory(stagingDirectory, model); + await writeModelLock(stagingDirectory, model); + await options.validate?.(stagingDirectory, model); + await publishModelDirectory(stagingDirectory, directory); + published = true; + } finally { + if (!published) await fs.rm(stagingDirectory, { recursive: true, force: true }); + } + return directory; + }); +} + +async function withInstallLock(directory, callback) { + const lockPath = path.join( + path.dirname(directory), + `.${path.basename(directory)}${MODEL_INSTALL_LOCK_FILE}` + ); + const owner = { + formatVersion: 1, + pid: process.pid, + hostname: os.hostname(), + createdAt: new Date().toISOString(), + token: randomUUID() + }; + let handle; + let heartbeat; + try { + handle = await acquireInstallLock(lockPath, directory, owner); + heartbeat = setInterval( + () => { + const now = new Date(); + fs.utimes(lockPath, now, now).catch(() => {}); + }, + Math.min(INSTALL_LOCK_STALE_MS / 3, 60 * 1000) + ); + heartbeat.unref(); + return await callback(); + } finally { + clearInterval(heartbeat); + await handle?.close().catch(() => {}); + if (handle) await releaseInstallLock(lockPath, owner); + } +} + +async function acquireInstallLock(lockPath, directory, owner) { + try { + return await createInstallLock(lockPath, owner); + } catch (error) { + if (error.code !== 'EEXIST') throw error; + if (await recoverStaleInstallLock(lockPath)) { + return createInstallLock(lockPath, owner); + } + throw Object.assign( + new Error(`Embedding model directory ${directory} is already being provisioned`, { + cause: error + }), + { code: MODEL_PROVISIONING_IN_PROGRESS } + ); + } +} + +async function createInstallLock(lockPath, owner) { + let handle; + try { + handle = await fs.open(lockPath, 'wx', 0o600); + await handle.writeFile(`${JSON.stringify(owner)}\n`); + await handle.sync(); + return handle; + } catch (error) { + await handle?.close().catch(() => {}); + if (handle) await fs.unlink(lockPath).catch(() => {}); + throw error; + } +} + +async function recoverStaleInstallLock(lockPath) { + let contents; + let stat; + try { + [contents, stat] = await Promise.all([fs.readFile(lockPath, 'utf8'), fs.lstat(lockPath)]); + } catch (error) { + return error.code === 'ENOENT'; + } + + let owner; + try { + owner = JSON.parse(contents); + } catch { + if (Date.now() - stat.mtimeMs <= INSTALL_LOCK_STALE_MS) return false; + } + if (!isStaleInstallLock(owner, stat)) return false; + + const stalePath = `${lockPath}.${randomUUID()}.stale`; + try { + await fs.rename(lockPath, stalePath); + const movedContents = await fs.readFile(stalePath, 'utf8'); + if (movedContents !== contents) { + await fs.rename(stalePath, lockPath).catch(() => {}); + return false; + } + await fs.unlink(stalePath); + return true; + } catch (error) { + await fs.unlink(stalePath).catch(() => {}); + return error.code === 'ENOENT'; + } +} + +function isStaleInstallLock(owner, stat) { + if (!owner || typeof owner !== 'object') { + return Date.now() - stat.mtimeMs > INSTALL_LOCK_STALE_MS; + } + if (owner.hostname === os.hostname() && Number.isSafeInteger(owner.pid)) { + try { + process.kill(owner.pid, 0); + return false; + } catch (error) { + if (error.code === 'EPERM') return false; + if (error.code === 'ESRCH') return true; + return false; + } + } + return Date.now() - stat.mtimeMs > INSTALL_LOCK_STALE_MS; +} + +async function releaseInstallLock(lockPath, owner) { + try { + const currentOwner = JSON.parse(await fs.readFile(lockPath, 'utf8')); + if (currentOwner.token === owner.token) await fs.unlink(lockPath); + } catch (error) { + if (error.code !== 'ENOENT') throw error; + } +} + +async function createStagingDirectory(directory) { + const parent = path.dirname(directory); + const stagingDirectory = await fs.mkdtemp( + path.join(parent, `.${path.basename(directory)}.staging-`) + ); + await fs.chmod(stagingDirectory, PROVISIONED_DIRECTORY_MODE); + return stagingDirectory; +} + +async function publishModelDirectory(stagingDirectory, directory) { + const targetExists = await assertModelDirectory(directory); + let backupDirectory; + if (targetExists) { + backupDirectory = path.join( + path.dirname(directory), + `.${path.basename(directory)}.${randomUUID()}.backup` + ); + await fs.rename(directory, backupDirectory); + } + + try { + await fs.rename(stagingDirectory, directory); + } catch (error) { + if (backupDirectory) await fs.rename(backupDirectory, directory).catch(() => {}); + throw error; + } + if (backupDirectory) await fs.rm(backupDirectory, { recursive: true, force: true }); +} + +async function directoryHasEntries(directory) { + return (await fs.readdir(directory)).length > 0; +} + +async function makeModelDirectoryReadable(directory, model) { + await fs.chmod(directory, PROVISIONED_DIRECTORY_MODE); + for (const file of model.files) { + let current = directory; + for (const part of file.name.split('/').slice(0, -1)) { + current = path.join(current, part); + // eslint-disable-next-line no-await-in-loop + await fs.chmod(current, PROVISIONED_DIRECTORY_MODE); + } + // eslint-disable-next-line no-await-in-loop + await fs.chmod(path.join(directory, file.name), PROVISIONED_FILE_MODE); + } + await fs.chmod(path.join(directory, MODEL_LOCK_FILE), PROVISIONED_FILE_MODE); +} + async function loadModelAndTokenizer(modelDir, model) { + const [{ Tokenizer }, { InferenceSession }] = await Promise.all([ + import('@huggingface/tokenizers'), + import('./InferenceSession.js') + ]); const modelPath = path.join(modelDir, fileForRole(model, 'model').name); const tokenizerPath = path.join(modelDir, fileForRole(model, 'tokenizer').name); const tokenizerConfigPath = path.join(modelDir, fileForRole(model, 'tokenizerConfig').name); @@ -239,20 +721,28 @@ async function loadModelAndTokenizer(modelDir, model) { fs.readFile(tokenizerPath, 'utf8').then(JSON.parse), fs.readFile(tokenizerConfigPath, 'utf8').then(JSON.parse) ]); + const tokenizer = new Tokenizer(tokenizerJson, tokenizerConfig); return { session: await InferenceSession.create(modelPath), - tokenizer: new Tokenizer(tokenizerJson, tokenizerConfig) + tokenizer }; } export { + MODEL_LOCK_FILE, + MODEL_PROVISIONING_IN_PROGRESS, + assertSafeRepository, downloadFile, downloadModelIfNeeded, - artifactSetDigest, fileForRole, - getModelCacheDir, + getModelDirectory, + getModelRoot, isValidFile, loadModelAndTokenizer, + modelDescriptorDigest, + provisionModel, + readModelLock, + verifyModelDirectory, validateModelDescriptor }; diff --git a/package.json b/package.json index c696aaa..701bb9c 100644 --- a/package.json +++ b/package.json @@ -8,8 +8,14 @@ "author": "SAP SE (https://www.sap.com)", "homepage": "https://cap.cloud.sap/", "main": "cds-plugin.js", + "bin": { + "cds-ai": "bin/cds-ai.js" + }, "scripts": { "lint": "npx -y eslint@10 .", + "test:model:provision": "node bin/cds-ai.js install-model Xenova/all-MiniLM-L6-v2", + "pretest": "npm run test:model:provision", + "pretest:hybrid": "npm run test:model:provision", "test": "node --test tests/*.test.js", "test:hybrid": "cds bind --exec -- node --test tests/*.test.js tests/integration/*.test.js", "format": "npx -y prettier@3 . --write && format-cds -f", @@ -17,6 +23,7 @@ }, "files": [ "CHANGELOG.md", + "bin", "lib", "srv" ], diff --git a/tests/model-discovery.test.js b/tests/model-discovery.test.js new file mode 100644 index 0000000..e6020e2 --- /dev/null +++ b/tests/model-discovery.test.js @@ -0,0 +1,326 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { describe, test } from 'node:test'; + +import { discoverModel } from '../lib/vector_embedding/model-discovery.js'; + +const REVISION = '1'.repeat(40); +const BASE_REVISION = '2'.repeat(40); +const REPOSITORY = 'example/embedding-model'; +const BASE_REPOSITORY = 'sentence-transformers/base-model'; + +describe('Hugging Face model discovery', () => { + test('creates a validated descriptor from exact artifacts and Sentence Transformers metadata', async () => { + const routes = modelRoutes({ + tokenizer: { truncation: { max_length: 96 } }, + tokenizerConfig: { model_max_length: 128 }, + config: { hidden_size: 384, max_position_embeddings: 512 }, + sentenceConfig: { max_seq_length: 256 }, + normalize: true + }); + + const descriptor = await discoverModel(REPOSITORY, { fetchImpl: createFetch(routes) }); + + assert.equal(descriptor.repository, REPOSITORY); + assert.equal(descriptor.revision, REVISION); + assert.equal(descriptor.dimensions, 384); + assert.equal(descriptor.maxLength, 96); + assert.deepEqual(descriptor.output, { + name: 'last_hidden_state', + pooling: 'mean', + normalize: true + }); + assert.deepEqual( + descriptor.files.map(({ role, name, path }) => ({ role, name, path })), + [ + { role: 'model', name: 'model.onnx', path: 'onnx/model.onnx' }, + { role: 'tokenizer', name: 'tokenizer.json', path: 'tokenizer.json' }, + { + role: 'tokenizerConfig', + name: 'tokenizer_config.json', + path: 'tokenizer_config.json' + }, + { role: 'auxiliary', name: 'config.json', path: 'config.json' } + ] + ); + const tokenizerFile = descriptor.files.find(({ role }) => role === 'tokenizer'); + assert.equal( + tokenizerFile.sha256, + digest(routes[fileUrl(REPOSITORY, REVISION, 'tokenizer.json')]) + ); + assert.equal( + tokenizerFile.size, + routes[fileUrl(REPOSITORY, REVISION, 'tokenizer.json')].length + ); + }); + + test('follows a pinned base_model for semantics and Sentence Transformers max length', async () => { + const routes = modelRoutes({ + tokenizer: { truncation: null }, + tokenizerConfig: { model_max_length: 128 }, + config: { hidden_size: 768, max_position_embeddings: 512 }, + modules: false, + baseModel: BASE_REPOSITORY + }); + addBaseModelRoutes(routes, { + sentenceConfig: { max_seq_length: 64 }, + pooling: 'cls', + normalize: false + }); + + const requested = []; + const descriptor = await discoverModel(REPOSITORY, { + fetchImpl: createFetch(routes, requested) + }); + + assert.equal(descriptor.maxLength, 64); + assert.equal(descriptor.output.pooling, 'cls'); + assert.equal(descriptor.output.normalize, false); + assert.ok( + requested.includes(apiUrl(BASE_REPOSITORY)), + 'the base repository is resolved through the model API' + ); + assert.ok( + requested.some((url) => url.includes(`/resolve/${BASE_REVISION}/modules.json`)), + 'base-model metadata is read from its immutable revision' + ); + }); + + test('falls back from tokenizer configuration to model configuration for max length', async () => { + const tokenizerRoutes = modelRoutes({ + tokenizer: { truncation: null }, + tokenizerConfig: { model_max_length: 192 }, + config: { hidden_size: 32, max_position_embeddings: 256 }, + sentenceConfig: undefined + }); + assert.equal( + (await discoverModel(REPOSITORY, { fetchImpl: createFetch(tokenizerRoutes) })).maxLength, + 192 + ); + + const configRoutes = modelRoutes({ + tokenizer: { truncation: null }, + tokenizerConfig: { model_max_length: 1e30 }, + config: { hidden_size: 32, max_position_embeddings: 256 }, + sentenceConfig: undefined + }); + assert.equal( + (await discoverModel(REPOSITORY, { fetchImpl: createFetch(configRoutes) })).maxLength, + 256 + ); + }); + + test('uses the lowest declared tokenizer and model input limit', async () => { + const routes = modelRoutes({ + tokenizer: { truncation: null }, + tokenizerConfig: { max_length: 128, model_max_length: 512 }, + config: { hidden_size: 32, max_position_embeddings: 256 }, + sentenceConfig: { max_seq_length: 384 } + }); + + assert.equal( + (await discoverModel(REPOSITORY, { fetchImpl: createFetch(routes) })).maxLength, + 128 + ); + }); + + test('includes external ONNX data files', async () => { + const routes = modelRoutes({ externalData: true }); + const descriptor = await discoverModel(REPOSITORY, { fetchImpl: createFetch(routes) }); + + assert.deepEqual( + descriptor.files.find(({ path }) => path === 'onnx/model.onnx_data'), + { + role: 'auxiliary', + name: 'model.onnx_data', + path: 'onnx/model.onnx_data', + size: routes[fileUrl(REPOSITORY, REVISION, 'onnx/model.onnx_data')].length, + sha256: digest(routes[fileUrl(REPOSITORY, REVISION, 'onnx/model.onnx_data')]) + } + ); + }); + + test('rejects missing exact artifacts and ambiguous runtime semantics', async () => { + const missing = modelRoutes({}); + const info = JSON.parse(missing[apiUrl(REPOSITORY)].toString()); + info.siblings = info.siblings.filter(({ rfilename }) => rfilename !== 'onnx/model.onnx'); + missing[apiUrl(REPOSITORY)] = json(info); + await assert.rejects( + discoverModel(REPOSITORY, { fetchImpl: createFetch(missing) }), + /exact file 'onnx\/model\.onnx'/ + ); + + const ambiguous = modelRoutes({ pooling: ['mean', 'cls'] }); + await assert.rejects( + discoverModel(REPOSITORY, { fetchImpl: createFetch(ambiguous) }), + /Unsupported or ambiguous Sentence Transformers pooling/ + ); + + const invalidOrder = modelRoutes({ moduleOrder: ['Pooling', 'Transformer'] }); + await assert.rejects( + discoverModel(REPOSITORY, { fetchImpl: createFetch(invalidOrder) }), + /unambiguous pooling pipeline/ + ); + + const conflicting = modelRoutes({ pooling: 'mean', poolingMode: 'cls' }); + await assert.rejects( + discoverModel(REPOSITORY, { fetchImpl: createFetch(conflicting) }), + /Unsupported or ambiguous Sentence Transformers pooling/ + ); + }); + + test('downloads an artifact to derive integrity when Hugging Face has no LFS metadata', async () => { + const routes = modelRoutes({ modelMetadata: false }); + const descriptor = await discoverModel(REPOSITORY, createFetch(routes)); + const model = descriptor.files.find(({ role }) => role === 'model'); + const contents = routes[fileUrl(REPOSITORY, REVISION, 'onnx/model.onnx')]; + assert.equal(model.size, contents.length); + assert.equal(model.sha256, digest(contents)); + }); +}); + +function modelRoutes(options = {}) { + const tokenizer = json(options.tokenizer ?? { truncation: { max_length: 96 } }); + const tokenizerConfig = json(options.tokenizerConfig ?? { model_max_length: 128 }); + const configValue = options.config ?? { hidden_size: 384, max_position_embeddings: 512 }; + if (options.externalData) { + configValue['transformers.js_config'] = { + use_external_data_format: { 'model.onnx': 1 } + }; + } + const config = json(configValue); + const model = Buffer.from('fake onnx model'); + const moduleTypes = options.moduleOrder ?? [ + 'Transformer', + 'Pooling', + ...(options.normalize === false ? [] : ['Normalize']) + ]; + const modules = json( + moduleTypes.map((type, index) => ({ + idx: index, + name: String(index), + path: type === 'Pooling' ? '1_Pooling' : '', + type: `sentence_transformers.models.${type}` + })) + ); + const pooling = json(poolingConfig(options.pooling ?? 'mean', options.poolingMode)); + const files = { + 'onnx/model.onnx': model, + 'tokenizer.json': tokenizer, + 'tokenizer_config.json': tokenizerConfig, + 'config.json': config + }; + if (options.externalData) files['onnx/model.onnx_data'] = Buffer.from('external weights'); + if (options.modules !== false) { + files['modules.json'] = modules; + files['1_Pooling/config.json'] = pooling; + if (options.sentenceConfig !== undefined) { + files['sentence_bert_config.json'] = json(options.sentenceConfig); + } else if (!Object.hasOwn(options, 'sentenceConfig')) { + files['sentence_bert_config.json'] = json({ max_seq_length: 256 }); + } + } + const siblings = Object.entries(files).map(([rfilename, contents]) => ({ + rfilename, + ...(rfilename === 'onnx/model.onnx' && options.modelMetadata !== false + ? { size: contents.length, lfs: { size: contents.length, sha256: digest(contents) } } + : {}) + })); + return { + [apiUrl(REPOSITORY)]: json({ + sha: REVISION, + siblings, + ...(options.baseModel ? { cardData: { base_model: options.baseModel } } : {}) + }), + ...Object.fromEntries( + Object.entries(files).map(([name, contents]) => [ + fileUrl(REPOSITORY, REVISION, name), + contents + ]) + ) + }; +} + +function addBaseModelRoutes(routes, options = {}) { + const modules = json([ + { + idx: 0, + path: '', + type: 'sentence_transformers.models.Transformer' + }, + { + idx: 1, + path: '1_Pooling', + type: 'sentence_transformers.models.Pooling' + }, + ...(options.normalize + ? [{ idx: 2, path: '2_Normalize', type: 'sentence_transformers.models.Normalize' }] + : []) + ]); + const pooling = json(poolingConfig(options.pooling ?? 'mean')); + const sentenceConfig = json(options.sentenceConfig ?? { max_seq_length: 128 }); + const files = { + 'modules.json': modules, + '1_Pooling/config.json': pooling, + 'sentence_bert_config.json': sentenceConfig + }; + routes[apiUrl(BASE_REPOSITORY)] = json({ + sha: BASE_REVISION, + siblings: Object.keys(files).map((rfilename) => ({ rfilename })) + }); + for (const [name, contents] of Object.entries(files)) { + routes[fileUrl(BASE_REPOSITORY, BASE_REVISION, name)] = contents; + } +} + +function poolingConfig(pooling, poolingMode) { + const enabled = Array.isArray(pooling) ? pooling : [pooling]; + return { + ...(poolingMode ? { pooling_mode: poolingMode } : {}), + pooling_mode_cls_token: enabled.includes('cls'), + pooling_mode_mean_tokens: enabled.includes('mean'), + pooling_mode_max_tokens: enabled.includes('max'), + pooling_mode_mean_sqrt_len_tokens: false, + pooling_mode_weightedmean_tokens: false, + pooling_mode_lasttoken: false + }; +} + +function createFetch(routes, requested = []) { + return async (input) => { + const url = String(input); + requested.push(url); + const contents = routes[url]; + if (!contents) return response(Buffer.alloc(0), 404); + return response(contents, 200); + }; +} + +function response(contents, status) { + return { + ok: status >= 200 && status < 300, + status, + async json() { + return JSON.parse(contents.toString()); + }, + async arrayBuffer() { + return contents; + } + }; +} + +function apiUrl(repository) { + return `https://huggingface.co/api/models/${repository}?blobs=true`; +} + +function fileUrl(repository, revision, name) { + return `https://huggingface.co/${repository}/resolve/${revision}/${name}`; +} + +function json(value) { + return Buffer.from(JSON.stringify(value)); +} + +function digest(value) { + return createHash('sha256').update(value).digest('hex'); +} diff --git a/tests/model-provisioning.test.js b/tests/model-provisioning.test.js new file mode 100644 index 0000000..01c7d0d --- /dev/null +++ b/tests/model-provisioning.test.js @@ -0,0 +1,598 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, test } from 'node:test'; +import { runModelCommand } from '../lib/vector_embedding/cli.js'; +import { resolveEmbeddingModel } from '../lib/vector_embedding/embedding.js'; +import { + MODEL_LOCK_FILE, + getModelDirectory, + getModelRoot, + provisionModel, + readModelLock, + verifyModelDirectory +} from '../lib/vector_embedding/model-utils.js'; + +const temporaryDirectories = []; + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => fs.rm(directory, { recursive: true, force: true })) + ); +}); + +describe('runtime model configuration', () => { + test('accepts only model and directory in runtime configuration', async () => { + const model = fixtureModel(Buffer.from('configuration fixture')); + await assert.rejects( + resolveEmbeddingModel(), + /cds\.env\.requires\.db\.embedding\.model must be a non-empty string/ + ); + await assert.rejects( + resolveEmbeddingModel({}), + /cds\.env\.requires\.db\.embedding\.model must be a non-empty string/ + ); + await assert.rejects( + resolveEmbeddingModel(model.repository), + /embedding must be an object with model and optional directory/ + ); + await assert.rejects( + resolveEmbeddingModel({ ...model }), + /Only model and directory are supported/ + ); + await assert.rejects( + resolveEmbeddingModel({ model: model.repository, directory: '' }), + /embedding\.directory must be a non-empty string/ + ); + }); +}); + +describe('explicit model provisioning', () => { + test('downloads, verifies, and locks a model idempotently', async () => { + const directory = await createTemporaryDirectory(); + const content = Buffer.from('verified model fixture'); + const model = fixtureModel(content); + const requestedUrls = []; + const fetchImpl = createFetch(content, requestedUrls); + + await provisionModel(model, { directory, fetchImpl }); + await provisionModel(model, { directory, fetchImpl }); + + assert.deepEqual( + requestedUrls, + model.files.map( + (file) => `https://huggingface.co/example/model/resolve/${model.revision}/${file.path}` + ) + ); + assert.deepEqual(await readModelLock(directory), model); + assert.deepEqual(JSON.parse(await fs.readFile(path.join(directory, MODEL_LOCK_FILE), 'utf8')), { + ...model, + formatVersion: 1 + }); + assert.deepEqual((await fs.readdir(directory)).sort(), [ + MODEL_LOCK_FILE, + 'model.onnx', + 'tokenizer.json', + 'tokenizer_config.json' + ]); + const modes = await Promise.all( + [MODEL_LOCK_FILE, ...model.files.map(({ name }) => name)].map(async (file) => + fs.stat(path.join(directory, file)).then(({ mode }) => mode & 0o777) + ) + ); + assert.deepEqual(modes, new Array(modes.length).fill(0o644)); + }); + + test('restores readable permissions on already valid artifacts', async () => { + const directory = await createTemporaryDirectory(); + const content = Buffer.from('readable model fixture'); + const model = fixtureModel(content); + const modelPath = path.join(directory, model.files[0].name); + await provisionModel(model, { directory, fetchImpl: createFetch(content) }); + await fs.chmod(modelPath, 0o600); + + await provisionModel(model, { directory, fetchImpl: createFetch(content) }); + + assert.equal((await fs.stat(modelPath)).mode & 0o777, 0o644); + }); + + test('makes newly provisioned directories traversable across runtime users', async () => { + const parent = await createTemporaryDirectory(); + const modelsDirectory = path.join(parent, 'models'); + const directory = path.join(modelsDirectory, 'custom'); + const content = Buffer.from('directory mode fixture'); + const baseModel = fixtureModel(content); + const model = { + ...baseModel, + files: baseModel.files.map((file, index) => + index === 0 ? { ...file, name: 'onnx/model.onnx' } : file + ) + }; + const originalUmask = process.umask(0o077); + try { + await provisionModel(model, { directory, fetchImpl: createFetch(content) }); + } finally { + process.umask(originalUmask); + } + + assert.equal((await fs.stat(modelsDirectory)).mode & 0o777, 0o755); + assert.equal((await fs.stat(directory)).mode & 0o777, 0o755); + assert.equal((await fs.stat(path.join(directory, 'onnx'))).mode & 0o777, 0o755); + assert.equal((await fs.stat(path.join(directory, 'onnx/model.onnx'))).mode & 0o777, 0o644); + assert.equal((await fs.stat(path.join(directory, MODEL_LOCK_FILE))).mode & 0o777, 0o644); + }); + + test('does not repurpose a directory locked to a different descriptor', async () => { + const directory = await createTemporaryDirectory(); + const content = Buffer.from('locked model fixture'); + const model = fixtureModel(content); + await provisionModel(model, { directory, fetchImpl: createFetch(content) }); + + await assert.rejects( + provisionModel({ ...model, repository: 'example/other-model' }, { directory }), + /locked to a different model descriptor/ + ); + await assert.rejects( + provisionModel({ ...model, dimensions: model.dimensions + 1 }, { directory }), + /locked to a different model descriptor/ + ); + assert.deepEqual(await readModelLock(directory), model); + }); + + test('serializes provisioning attempts for the same directory', async () => { + const directory = await createTemporaryDirectory(); + const content = Buffer.from('concurrent model fixture'); + const model = fixtureModel(content); + let releaseDownload; + let signalDownloadStarted; + const downloadStarted = new Promise((resolve) => { + signalDownloadStarted = resolve; + }); + const waitForRelease = new Promise((resolve) => { + releaseDownload = resolve; + }); + let firstRequest = true; + const fetchImpl = async () => { + if (firstRequest) { + firstRequest = false; + signalDownloadStarted(); + await waitForRelease; + } + return new Response(content); + }; + + const first = provisionModel(model, { directory, fetchImpl }); + await downloadStarted; + await assert.rejects( + provisionModel(model, { directory, fetchImpl }), + /already being provisioned/ + ); + releaseDownload(); + await first; + }); + + test('recovers a stale install lock owned by a terminated local process', async () => { + const parent = await createTemporaryDirectory(); + const directory = path.join(parent, 'model'); + const installLock = path.join(parent, '.model.embedding.install.lock'); + const content = Buffer.from('stale lock fixture'); + const model = fixtureModel(content); + await fs.writeFile( + installLock, + JSON.stringify({ + formatVersion: 1, + pid: 99999999, + hostname: os.hostname(), + createdAt: '2000-01-01T00:00:00.000Z', + token: 'stale-owner' + }) + ); + + await provisionModel(model, { directory, fetchImpl: createFetch(content) }); + + await assert.rejects(fs.access(installLock)); + assert.deepEqual(await readModelLock(directory), model); + }); + + test('recovers a stale install lock truncated by an interrupted write', async () => { + const parent = await createTemporaryDirectory(); + const directory = path.join(parent, 'model'); + const installLock = path.join(parent, '.model.embedding.install.lock'); + const content = Buffer.from('truncated lock fixture'); + const model = fixtureModel(content); + await fs.writeFile(installLock, '{'); + const staleTime = new Date('2000-01-01T00:00:00.000Z'); + await fs.utimes(installLock, staleTime, staleTime); + + await provisionModel(model, { directory, fetchImpl: createFetch(content) }); + + await assert.rejects(fs.access(installLock)); + assert.deepEqual(await readModelLock(directory), model); + }); + + test('rejects symlinked artifact path components', async () => { + const directory = await createTemporaryDirectory(); + const outside = await createTemporaryDirectory(); + const content = Buffer.from('symlink model fixture'); + const baseModel = fixtureModel(content); + const model = { + ...baseModel, + files: baseModel.files.map((file, index) => + index === 0 ? { ...file, name: 'nested/model.onnx' } : file + ) + }; + await fs.symlink(outside, path.join(directory, 'nested'), 'dir'); + + await assert.rejects( + provisionModel(model, { directory, fetchImpl: createFetch(content) }), + /must not contain symbolic links/ + ); + assert.deepEqual(await fs.readdir(outside), []); + }); + + test('rejects a symlinked or replaced model directory', async () => { + const parent = await createTemporaryDirectory(); + const directory = path.join(parent, 'model'); + const outside = await createTemporaryDirectory(); + const content = Buffer.from('root symlink fixture'); + const model = fixtureModel(content); + await fs.symlink(outside, directory, 'dir'); + + await assert.rejects( + provisionModel(model, { directory, fetchImpl: createFetch(content) }), + /model directory must not be a symbolic link/ + ); + assert.deepEqual(await fs.readdir(outside), []); + }); + + test('canonicalizes symlinked ancestor directories before provisioning', async () => { + const parent = await createTemporaryDirectory(); + const outside = await createTemporaryDirectory(); + const modelsDirectory = path.join(parent, 'models'); + const requestedDirectory = path.join(modelsDirectory, 'custom'); + const content = Buffer.from('ancestor symlink fixture'); + const model = fixtureModel(content); + await fs.symlink(outside, modelsDirectory, 'dir'); + + const directory = await provisionModel(model, { + directory: requestedDirectory, + fetchImpl: createFetch(content) + }); + + assert.equal(directory, path.join(await fs.realpath(outside), 'custom')); + assert.deepEqual(await readModelLock(directory), model); + }); + + test('publishes from staging and detects replacement of the target during download', async () => { + const parent = await createTemporaryDirectory(); + const directory = path.join(parent, 'model'); + const outside = await createTemporaryDirectory(); + const content = Buffer.from('target replacement fixture'); + const model = fixtureModel(content); + let replaced = false; + const fetchImpl = async () => { + if (!replaced) { + replaced = true; + await fs.symlink(outside, directory, 'dir'); + } + return new Response(content); + }; + + await assert.rejects( + provisionModel(model, { directory, fetchImpl }), + /model directory must not be a symbolic link/ + ); + assert.deepEqual(await fs.readdir(outside), []); + }); + + test('writes the lock only after every artifact passes verification', async () => { + const directory = await createTemporaryDirectory(); + const content = Buffer.from('expected model fixture'); + const model = fixtureModel(content); + + await assert.rejects( + provisionModel(model, { + directory, + fetchImpl: async () => new Response(Buffer.from('invalid')) + }), + /Invalid size|Invalid SHA-256/ + ); + await assert.rejects(fs.access(path.join(directory, MODEL_LOCK_FILE))); + }); + + test('validates the runtime before publishing a newly installed model', async () => { + const parent = await createTemporaryDirectory(); + const directory = path.join(parent, 'model'); + const content = Buffer.from('runtime validation fixture'); + const model = fixtureModel(content); + let stagedDirectory; + + await assert.rejects( + provisionModel(model, { + directory, + fetchImpl: createFetch(content), + validate(candidate) { + stagedDirectory = candidate; + throw new Error('incompatible ONNX runtime'); + } + }), + /incompatible ONNX runtime/ + ); + + assert.notEqual(stagedDirectory, directory); + await assert.rejects(fs.access(directory)); + }); + + test('fails verification instead of downloading missing runtime files', async () => { + const directory = await createTemporaryDirectory(); + const model = fixtureModel(Buffer.from('fixture')); + + await assert.rejects( + verifyModelDirectory(directory, model), + new RegExp(`Embedding model is not provisioned.*${escapeRegExp(directory)}`, 's') + ); + assert.deepEqual(await fs.readdir(directory), []); + }); + + test('downloads a missing model into the project-local default directory and reuses it', async () => { + const root = await createTemporaryDirectory(); + const content = Buffer.from('lazy download fixture'); + const model = fixtureModel(content); + const requestedUrls = []; + const warnings = []; + let discoveries = 0; + const options = { + root, + fetchImpl: createFetch(content, requestedUrls), + discover(name) { + discoveries++; + assert.equal(name, model.repository); + return model; + }, + validate: async () => {}, + warn: (message) => warnings.push(message) + }; + + const first = await resolveEmbeddingModel({ model: model.repository }, options); + const expectedDirectory = getModelDirectory(getModelRoot(undefined, root), model.repository); + + assert.equal(first.model, model); + assert.equal(first.modelDir, expectedDirectory); + assert.equal(discoveries, 1); + assert.equal(warnings.length, 1); + assert.match(warnings[0], /Downloading it now; application startup may be delayed/); + assert.equal(requestedUrls.length, model.files.length); + assert.deepEqual(await readModelLock(expectedDirectory), model); + + const second = await resolveEmbeddingModel({ model: model.repository }, options); + assert.equal(second.modelDir, expectedDirectory); + assert.equal(discoveries, 1); + assert.equal(warnings.length, 1); + assert.equal(requestedUrls.length, model.files.length); + }); + + test('waits for concurrent ad-hoc provisioning and reuses the completed download', async () => { + const root = await createTemporaryDirectory(); + const content = Buffer.from('concurrent lazy download fixture'); + const model = fixtureModel(content); + const requestedUrls = []; + const warnings = []; + let releaseDownload; + let signalDownloadStarted; + let firstRequest = true; + const downloadStarted = new Promise((resolve) => { + signalDownloadStarted = resolve; + }); + const waitForRelease = new Promise((resolve) => { + releaseDownload = resolve; + }); + const fetchImpl = async (url) => { + requestedUrls.push(url); + if (firstRequest) { + firstRequest = false; + signalDownloadStarted(); + await waitForRelease; + } + return new Response(content, { + headers: { 'content-length': String(content.length) } + }); + }; + const options = { + root, + fetchImpl, + discover: () => model, + validate: async () => {}, + warn: (message) => warnings.push(message), + provisionRetryMs: 5, + provisionTimeoutMs: 1000 + }; + + const first = resolveEmbeddingModel({ model: model.repository }, options); + await downloadStarted; + const second = resolveEmbeddingModel({ model: model.repository }, options); + await new Promise((resolve) => setTimeout(resolve, 20)); + releaseDownload(); + + const resolved = await Promise.all([first, second]); + assert.equal(resolved[0].modelDir, resolved[1].modelDir); + assert.equal(warnings.length, 2); + assert.equal(requestedUrls.length, model.files.length); + }); + + test('keeps explicitly configured directories offline', async () => { + const root = await createTemporaryDirectory(); + let fetched = false; + await assert.rejects( + resolveEmbeddingModel( + { + model: 'example/model', + directory: './models/minilm' + }, + { + root, + fetchImpl: () => { + fetched = true; + throw new Error('explicit directories must not fetch'); + } + } + ), + /@cap-js\/ai install-model example\/model --directory \.\/models/ + ); + assert.equal(fetched, false); + }); + + test('resolves relative directories from cds.root and preserves absolute directories', async () => { + const root = await createTemporaryDirectory(); + const content = Buffer.from('directory resolution fixture'); + const model = fixtureModel(content); + const modelRoot = path.join(root, 'models'); + const modelDir = getModelDirectory(modelRoot, model.repository); + await provisionModel(model, { directory: modelDir, fetchImpl: createFetch(content) }); + + const relative = await resolveEmbeddingModel( + { model: model.repository, directory: './models' }, + { root } + ); + const absolute = await resolveEmbeddingModel( + { model: model.repository, directory: modelRoot }, + { root: await createTemporaryDirectory() } + ); + + assert.equal(relative.modelDir, modelDir); + assert.equal(absolute.modelDir, modelDir); + }); + + test('rejects a configured model name that does not match the provisioned lock', async () => { + const modelRoot = await createTemporaryDirectory(); + const content = Buffer.from('repository mismatch fixture'); + const model = fixtureModel(content); + const modelDir = getModelDirectory(modelRoot, 'example/other-model'); + await provisionModel(model, { directory: modelDir, fetchImpl: createFetch(content) }); + + await assert.rejects( + resolveEmbeddingModel({ model: 'example/other-model', directory: modelRoot }), + /contains example\/model, not example\/other-model/ + ); + }); + + test('gives models a model-name provisioning command', async () => { + const root = await createTemporaryDirectory(); + + await assert.rejects( + resolveEmbeddingModel({ model: 'example/custom', directory: './models/custom' }, { root }), + /@cap-js\/ai install-model example\/custom --directory \.\/models\/custom/ + ); + }); + + test('requires explicit lock recovery before reinstalling', async () => { + const modelRoot = await createTemporaryDirectory(); + const modelDir = getModelDirectory(modelRoot, 'example/model'); + await fs.mkdir(modelDir, { recursive: true }); + await fs.writeFile(path.join(modelDir, MODEL_LOCK_FILE), '{}'); + + await assert.rejects( + resolveEmbeddingModel({ model: 'example/model', directory: modelRoot }), + /Remove or replace the invalid lock explicitly, then run 'npx @cap-js\/ai install-model/ + ); + }); + + test('installs a model by name through the command API', async () => { + const root = await createTemporaryDirectory(); + const modelRoot = path.join(root, 'models'); + const content = Buffer.from('command fixture'); + const model = fixtureModel(content); + const output = []; + + await runModelCommand(['install-model', model.repository, '--directory', modelRoot], { + cwd: root, + discover: () => model, + fetchImpl: createFetch(content), + validate: async () => {}, + stdout: { write: (value) => output.push(value) } + }); + + const modelDir = getModelDirectory(modelRoot, model.repository); + assert.deepEqual(await readModelLock(modelDir), model); + assert.match(output.join(''), /Installed example\/model/); + assert.match(output.join(''), new RegExp(escapeRegExp(modelDir))); + }); + + test('installs into the project-local model cache when no directory is provided', async () => { + const root = await createTemporaryDirectory(); + const content = Buffer.from('default command fixture'); + const model = fixtureModel(content); + + await runModelCommand(['install-model', model.repository], { + cwd: root, + discover: () => model, + fetchImpl: createFetch(content), + validate: async () => {}, + stdout: { write() {} } + }); + + const modelDir = path.join(root, '.cds', 'models', 'example', 'model'); + assert.deepEqual(await readModelLock(modelDir), model); + }); + + test('requires a model name and accepts an optional cache root', async () => { + await assert.rejects( + runModelCommand(['install-model', '--directory', './models/custom']), + /Specify a model name/ + ); + await assert.rejects( + runModelCommand(['install-model', 'example/model', 'example/other']), + /Unexpected argument 'example\/other'/ + ); + }); +}); + +function createFetch(content, requestedUrls = []) { + return async (url) => { + requestedUrls.push(url); + return new Response(content, { + headers: { 'content-length': String(content.length) } + }); + }; +} + +async function createTemporaryDirectory() { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'cap-ai-provision-')); + temporaryDirectories.push(directory); + return directory; +} + +function fixtureModel(content) { + const sha256 = createHash('sha256').update(content).digest('hex'); + return { + repository: 'example/model', + revision: 'deadbeefdeadbeefdeadbeefdeadbeefdeadbeef', + dimensions: 2, + maxLength: 8, + files: [ + { role: 'model', name: 'model.onnx', path: 'onnx/model.onnx', size: content.length, sha256 }, + { + role: 'tokenizer', + name: 'tokenizer.json', + path: 'tokenizer.json', + size: content.length, + sha256 + }, + { + role: 'tokenizerConfig', + name: 'tokenizer_config.json', + path: 'tokenizer_config.json', + size: content.length, + sha256 + } + ], + output: { name: 'last_hidden_state', pooling: 'mean', normalize: true } + }; +} + +function escapeRegExp(value) { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/tests/vector-unit.test.js b/tests/vector-unit.test.js index 7162669..dc5bf45 100644 --- a/tests/vector-unit.test.js +++ b/tests/vector-unit.test.js @@ -4,6 +4,7 @@ import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { afterEach, describe, test } from 'node:test'; +import { InferenceSession } from '../lib/vector_embedding/InferenceSession.js'; import { createFeeds, createTokenizerState, @@ -11,15 +12,29 @@ import { tokenizeWithChunks } from '../lib/vector_embedding/embedding.js'; import { - artifactSetDigest, downloadFile, downloadModelIfNeeded, - getModelCacheDir, + getModelDirectory, + getModelRoot, validateModelDescriptor } from '../lib/vector_embedding/model-utils.js'; const temporaryDirectories = []; +test('disposes inference sessions at most once', async () => { + let disposals = 0; + const session = new InferenceSession({ + dispose() { + disposals++; + } + }); + + await session.dispose(); + await session.dispose(); + + assert.equal(disposals, 1); +}); + afterEach(async () => { await Promise.all( temporaryDirectories @@ -132,31 +147,60 @@ describe('model compatibility', () => { }), /safe relative path/ ); + assert.throws( + () => + validateModelDescriptor({ + ...model, + files: model.files.map((file, index) => + index === 0 ? { ...file, name: 'embedding.lock.json' } : file + ) + }), + /conflicts with provisioning metadata/ + ); + assert.throws( + () => + validateModelDescriptor({ + ...model, + files: model.files.map((file, index) => + index === 0 ? { ...file, name: 'EMBEDDING.LOCK.JSON' } : file + ) + }), + /conflicts with provisioning metadata/ + ); + assert.throws( + () => + validateModelDescriptor({ + ...model, + files: model.files.map((file, index) => { + if (index === 0) return { ...file, name: 'nested' }; + if (index === 1) return { ...file, name: 'nested/tokenizer.json' }; + return file; + }) + }), + /conflicts with another embedding file/ + ); }); }); -describe('model cache identity', () => { - test('preserves repository components and separates artifact variants', () => { - const root = path.join(path.sep, 'cache'); - const model = fixtureModel(Buffer.from('fixture')); - const reordered = { ...model, files: [...model.files].reverse() }; - const variant = { - ...model, - files: model.files.map((file, index) => - index === 0 ? { ...file, sha256: 'f'.repeat(64) } : file - ) - }; - const flattenedRepository = { ...model, repository: 'example_model' }; +describe('model directories', () => { + test('uses a project-local default and appends the model repository', () => { + const project = path.join(path.sep, 'project'); + const root = getModelRoot(undefined, project); + + assert.equal(root, path.join(project, '.cds', 'models')); + assert.equal(getModelDirectory(root, 'foo/bar'), path.join(root, 'foo', 'bar')); + }); + + test('resolves relative, absolute, and home-relative roots', () => { + const project = path.join(path.sep, 'project'); + const home = path.join(path.sep, 'home', 'user'); - const modelPath = getModelCacheDir(root, model); - assert.equal(modelPath, getModelCacheDir(root, reordered)); - assert.notEqual(modelPath, getModelCacheDir(root, variant)); - assert.notEqual(modelPath, getModelCacheDir(root, flattenedRepository)); + assert.equal(getModelRoot('./models', project, home), path.join(project, 'models')); assert.equal( - path.relative(root, modelPath).split(path.sep).slice(0, 3).join('/'), - `example/model/${model.revision}` + getModelRoot(path.join(path.sep, 'shared', 'models'), project, home), + path.join(path.sep, 'shared', 'models') ); - assert.equal(path.basename(modelPath), artifactSetDigest(model)); + assert.equal(getModelRoot('~/.cds/models', project, home), path.join(home, '.cds', 'models')); }); }); diff --git a/tests/vector.test.js b/tests/vector.test.js index 50b3fe2..89a92b5 100644 --- a/tests/vector.test.js +++ b/tests/vector.test.js @@ -2,12 +2,13 @@ import { after, before, describe, test } from 'node:test'; import assert from 'node:assert'; import cds from '@sap/cds'; import { initializeEmbedding, vector_embedding } from '../lib/vector_embedding/index.js'; -import { DEFAULT_MODEL } from '../lib/vector_embedding/embedding.js'; + +const MINILM_MODEL = 'Xenova/all-MiniLM-L6-v2'; let embeddingModule; before(async () => { - embeddingModule = await initializeEmbedding(); + embeddingModule = await initializeEmbedding({ model: MINILM_MODEL }); }); describe('Vector embedding function (standalone)', () => { @@ -113,7 +114,11 @@ describe('Vector embedding function (standalone)', () => { const result3 = vector_embedding('test', 'DOCUMENT', 'unknown_model'); const embedding3 = JSON.parse(result3); - assert.strictEqual(embedding3.length, 384, 'Unknown model should default to 384 dimensions'); + assert.strictEqual( + embedding3.length, + 384, + 'Compatibility identifiers should use the configured model dimensions' + ); }); test('retains the embedding module wrapper', () => { @@ -129,9 +134,20 @@ describe('Vector embedding function (standalone)', () => { describe('ai-sqlite integration', () => { let db; + test('requires an explicitly configured embedding model during startup', async () => { + await assert.rejects( + cds.connect.to('missing-embedding-model-db', { + kind: 'ai-sqlite', + credentials: { url: ':memory:' } + }), + /cds\.env\.requires\.db\.embedding\.model must be a non-empty string/ + ); + }); + before(async () => { db = await cds.connect.to('vector-db', { kind: 'ai-sqlite', + embedding: { model: MINILM_MODEL }, credentials: { url: ':memory:' } }); }); @@ -157,43 +173,16 @@ describe('ai-sqlite integration', () => { assert.strictEqual(row.embedding, null); }); - test('validates a model descriptor supplied through the service options', async () => { + test('rejects model descriptors supplied through the service options', async () => { await assert.rejects( cds.connect.to('invalid-vector-db', { kind: 'ai-sqlite', - embedding: { ...DEFAULT_MODEL, revision: 'main' }, + embedding: { model: MINILM_MODEL, revision: 'main' }, credentials: { url: ':memory:' } }), - /immutable 40-64 character commit hash/ + /Only model and directory are supported/ ); }); - - test('keeps custom model behavior scoped to its service', async () => { - const customDb = await cds.connect.to('unnormalized-vector-db', { - kind: 'ai-sqlite', - embedding: { - ...DEFAULT_MODEL, - output: { ...DEFAULT_MODEL.output, normalize: false } - }, - credentials: { url: ':memory:' } - }); - - try { - const [defaultRow] = await db.run( - `SELECT VECTOR_EMBEDDING('Hello world', 'DOCUMENT', 'SAP_GXY.20250407') AS embedding` - ); - const [customRow] = await customDb.run( - `SELECT VECTOR_EMBEDDING('Hello world', 'DOCUMENT', 'SAP_GXY.20250407') AS embedding` - ); - const defaultNorm = vectorNorm(JSON.parse(defaultRow.embedding)); - const customNorm = vectorNorm(JSON.parse(customRow.embedding)); - - assert.ok(Math.abs(defaultNorm - 1) < 1e-5); - assert.ok(Math.abs(customNorm - 1) > 1e-3); - } finally { - await customDb.disconnect(); - } - }); }); // Helper function to calculate cosine similarity between two vectors @@ -212,7 +201,3 @@ function cosineSimilarity(a, b) { return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)); } - -function vectorNorm(vector) { - return Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0)); -}