From fd34986dfd023e8cc60d337ddeda0d7b8512c305 Mon Sep 17 00:00:00 2001 From: Sebastian Van Syckel Date: Wed, 26 Aug 2026 14:09:17 +0200 Subject: [PATCH 1/6] feat: add explicit embedding model provisioning --- CHANGELOG.md | 2 +- README.md | 97 +++-- bin/cds-ai.js | 10 + lib/sqlite/AISQLiteService.js | 4 +- lib/vector_embedding/cli.js | 80 +++++ lib/vector_embedding/embedding.js | 138 +++++--- lib/vector_embedding/model-utils.js | 528 +++++++++++++++++++++++++++- lib/vector_embedding/models.js | 52 +++ package.json | 7 + tests/model-provisioning.test.js | 425 ++++++++++++++++++++++ tests/vector-unit.test.js | 32 ++ tests/vector.test.js | 5 + 12 files changed, 1281 insertions(+), 99 deletions(-) create mode 100755 bin/cds-ai.js create mode 100644 lib/vector_embedding/cli.js create mode 100644 lib/vector_embedding/models.js create mode 100644 tests/model-provisioning.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index a15e27c..793d0e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ ### Added - Add the `ai-sqlite` kind with a `VECTOR_EMBEDDING` function using ONNX Runtime and the `Xenova/all-MiniLM-L6-v2` model (384 dimensions) - - Downloads the pinned model revision on-demand from Hugging Face (~91MB), verifies its size and SHA-256, and caches it locally + - Adds `cds-ai model install` for explicit, checksum-verified model provisioning; application startup remains offline - Uses `@huggingface/tokenizers` and chunks long input without dropping per-chunk special tokens - Allows an explicit, checksum-verified compatible encoder model descriptor per `ai-sqlite` service - Supports both 3-parameter `(text, text_type, model_and_version)` and 4-parameter variants with `remote_source` diff --git a/README.md b/README.md index 2f2de63..8251935 100644 --- a/README.md +++ b/README.md @@ -219,6 +219,14 @@ npm add @cap-js/sqlite onnxruntime-node@1.20.1 `ai-sqlite` currently requires exactly `onnxruntime-node` 1.20.1 because synchronous SQLite functions need a version-specific native runtime API. +Provision the default model before starting the application: + +```sh +npx cds-ai model install Xenova/all-MiniLM-L6-v2 +``` + +The command downloads the pinned artifacts, verifies their sizes and SHA-256 checksums, and writes an `embedding.lock.json`. Application startup never downloads model files. + Select `ai-sqlite` for the database service: ```json @@ -252,7 +260,8 @@ SELECT.from('Books').columns` **Features:** - **Initialization**: The ONNX model is loaded when the `ai-sqlite` service starts -- **Verified cache**: The pinned model revision and artifact set are cached by default below the user's data directory; set `CDS_AI_MODEL_CACHE` to use a pre-provisioned cache root +- **Explicit provisioning**: Model artifacts are downloaded only by `cds-ai model install`; runtime initialization is offline +- **Verified cache**: The pinned model revision and artifact set are stored by default below the user's data directory; set `CDS_AI_MODEL_CACHE` to select another cache root - **Hugging Face tokenization**: Uses `@huggingface/tokenizers` and safely chunks text that exceeds the model limit - **Deterministic**: Same input always produces same output - **Normalized vectors**: MiniLM embeddings are L2-normalized; custom descriptors control this with `output.normalize` @@ -260,7 +269,52 @@ SELECT.from('Books').columns` #### Compatible custom encoder models -Configure a different model through the database service's `embedding` option. Models are not discovered dynamically: every artifact must belong to an immutable revision and have an expected size and SHA-256 checksum. +The built-in preset currently recognizes `Xenova/all-MiniLM-L6-v2`. For a compatible custom model, create an `embedding-model.json` descriptor. Models are not discovered dynamically: every artifact must belong to an immutable revision and have an expected size and SHA-256 checksum. + +```json +{ + "repository": "organization/model", + "revision": "0123456789abcdef0123456789abcdef01234567", + "dimensions": 768, + "maxLength": 512, + "files": [ + { + "role": "model", + "name": "model.onnx", + "path": "onnx/model.onnx", + "size": 123456789, + "sha256": "<64 lowercase hexadecimal characters>" + }, + { + "role": "tokenizer", + "name": "tokenizer.json", + "path": "tokenizer.json", + "size": 123456, + "sha256": "<64 lowercase hexadecimal characters>" + }, + { + "role": "tokenizerConfig", + "name": "tokenizer_config.json", + "path": "tokenizer_config.json", + "size": 1234, + "sha256": "<64 lowercase hexadecimal characters>" + } + ], + "output": { + "name": "last_hidden_state", + "pooling": "mean", + "normalize": true + } +} +``` + +Provision it into an application-managed directory: + +```sh +npx cds-ai model install --descriptor ./embedding-model.json --directory ./models/custom +``` + +Then point the service at that locked directory: ```json { @@ -269,38 +323,8 @@ Configure a different model through the database service's `embedding` option. M "db": { "kind": "ai-sqlite", "embedding": { - "repository": "organization/model", - "revision": "0123456789abcdef0123456789abcdef01234567", - "dimensions": 768, - "maxLength": 512, - "files": [ - { - "role": "model", - "name": "model.onnx", - "path": "onnx/model.onnx", - "size": 123456789, - "sha256": "<64 lowercase hexadecimal characters>" - }, - { - "role": "tokenizer", - "name": "tokenizer.json", - "path": "tokenizer.json", - "size": 123456, - "sha256": "<64 lowercase hexadecimal characters>" - }, - { - "role": "tokenizerConfig", - "name": "tokenizer_config.json", - "path": "tokenizer_config.json", - "size": 1234, - "sha256": "<64 lowercase hexadecimal characters>" - } - ], - "output": { - "name": "last_hidden_state", - "pooling": "mean", - "normalize": true - } + "model": "organization/model", + "directory": "./models/custom" } } } @@ -308,12 +332,15 @@ Configure a different model through the database service's `embedding` option. M } ``` +Relative directories are resolved from the CAP project root. Provisioning canonicalizes symlinked parent directories and rejects a model directory that is itself a symlink. For production, provision the directory while building the application image or mount it read-only. The inline descriptor form from earlier versions remains supported and uses the configured cache root. + Compatible models must accept `input_ids` and may additionally accept `attention_mask` and `token_type_ids`, all as `int64` tensors. Their configured float32 or float64 output must support `mean` or `cls` pooling from `[1, sequence, dimensions]`, or `none` for an already pooled `[dimensions]` or `[1, dimensions]` tensor. Additional pinned ONNX data files can use the `auxiliary` role. Startup probes the model and rejects incompatible input names, output names, types, shapes, or dimensions. **Error Handling:** - Starting `ai-sqlite` fails if the ONNX model cannot be initialized -- Downloads are time-limited and accepted only when their expected size and SHA-256 match +- Starting `ai-sqlite` fails with a provisioning command if model artifacts are missing or fail integrity checks +- Provisioning downloads are time-limited and accepted only when their expected size and SHA-256 match - Throws if embedding generation fails ## Test the plugin locally diff --git a/bin/cds-ai.js b/bin/cds-ai.js new file mode 100755 index 0000000..788d63f --- /dev/null +++ b/bin/cds-ai.js @@ -0,0 +1,10 @@ +#!/usr/bin/env node + +import { runModelCommand } from '../lib/vector_embedding/cli.js'; + +try { + await runModelCommand(process.argv.slice(2)); +} catch (error) { + process.stderr.write(`${error.message}\n`); + process.exitCode = 1; +} diff --git a/lib/sqlite/AISQLiteService.js b/lib/sqlite/AISQLiteService.js index cb5dc7e..07212ac 100644 --- a/lib/sqlite/AISQLiteService.js +++ b/lib/sqlite/AISQLiteService.js @@ -6,7 +6,9 @@ const LOG = cds.log('@cap-js/ai'); export default class AISQLiteService extends SQLiteService { async init() { - this._embeddingRuntime = await createEmbeddingRuntime(this.options.embedding); + this._embeddingRuntime = await createEmbeddingRuntime(this.options.embedding, { + root: cds.root + }); LOG.info('Vector embedding ONNX model initialized'); return super.init(); } diff --git a/lib/vector_embedding/cli.js b/lib/vector_embedding/cli.js new file mode 100644 index 0000000..ea889c9 --- /dev/null +++ b/lib/vector_embedding/cli.js @@ -0,0 +1,80 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { + getModelCacheDir, + getModelCacheRoot, + provisionModel, + validateModelDescriptor +} from './model-utils.js'; +import { resolveModelPreset } from './models.js'; + +const HELP = `Usage: + cds-ai model install [--directory ] + cds-ai model install --descriptor [--directory ] + +Options: + --directory Install into an application-managed directory + --descriptor Read a compatible custom model descriptor from JSON + --help Show this help +`; + +async function runModelCommand(argv, options = {}) { + const { cwd = process.cwd(), env = process.env, fetchImpl, stdout = process.stdout } = options; + const command = parseArguments(argv); + if (command.help) { + stdout.write(HELP); + return; + } + + const model = command.descriptor + ? await readDescriptor(path.resolve(cwd, command.descriptor)) + : resolveModelPreset(command.model); + const directory = command.directory + ? path.resolve(cwd, command.directory) + : getModelCacheDir(getModelCacheRoot(env), model); + + await provisionModel(model, { directory, env, fetchImpl }); + stdout.write(`Provisioned ${model.repository} in ${directory}\n`); +} + +function parseArguments(argv) { + if (argv.length === 0 || argv.includes('--help') || argv.includes('-h')) return { help: true }; + if (argv[0] !== 'model' || argv[1] !== 'install') { + throw new Error(`Unsupported command.\n\n${HELP}`); + } + + let model; + let descriptor; + let directory; + for (let index = 2; index < argv.length; index++) { + const argument = argv[index]; + if (argument === '--descriptor' || argument === '--directory') { + const value = argv[++index]; + if (!value || value.startsWith('--')) throw new Error(`${argument} requires a value`); + if (argument === '--descriptor') descriptor = value; + else directory = value; + continue; + } + if (argument.startsWith('-')) throw new Error(`Unknown option '${argument}'`); + if (model) throw new Error(`Unexpected argument '${argument}'`); + model = argument; + } + + if (descriptor && model) throw new Error('Specify either a model name or --descriptor, not both'); + if (!descriptor && !model) throw new Error('Specify a model name or --descriptor'); + return { descriptor, directory, model }; +} + +async function readDescriptor(file) { + let descriptor; + try { + descriptor = JSON.parse(await fs.readFile(file, 'utf8')); + } catch (error) { + throw new Error(`Cannot read embedding model descriptor at ${file}: ${error.message}`, { + cause: error + }); + } + return validateModelDescriptor(descriptor); +} + +export { HELP, parseArguments, runModelCommand }; diff --git a/lib/vector_embedding/embedding.js b/lib/vector_embedding/embedding.js index 5c8c10f..003c86d 100644 --- a/lib/vector_embedding/embedding.js +++ b/lib/vector_embedding/embedding.js @@ -1,54 +1,20 @@ -import os from 'os'; import path from 'path'; import { Tensor } from './InferenceSession.js'; import { - downloadModelIfNeeded, getModelCacheDir, + getModelCacheRoot, loadModelAndTokenizer, + modelDescriptorDigest, + readModelLock, + verifyModelDirectory, validateModelDescriptor } from './model-utils.js'; +import { DEFAULT_MODEL, findModelPreset, resolveModelPreset } from './models.js'; const STANDARD_INPUT_NAMES = new Set(['input_ids', 'attention_mask', 'token_type_ids']); -const DEFAULT_MODEL = Object.freeze({ - repository: 'Xenova/all-MiniLM-L6-v2', - revision: '751bff37182d3f1213fa05d7196b954e230abad9', - dimensions: 384, - maxLength: 128, - files: Object.freeze([ - Object.freeze({ - role: 'model', - name: 'model.onnx', - path: 'onnx/model.onnx', - size: 90387606, - sha256: '759c3cd2b7fe7e93933ad23c4c9181b7396442a2ed746ec7c1d46192c469c46e' - }), - Object.freeze({ - role: 'tokenizer', - name: 'tokenizer.json', - path: 'tokenizer.json', - size: 711661, - sha256: 'da0e79933b9ed51798a3ae27893d3c5fa4a201126cef75586296df9b4d2c62a0' - }), - Object.freeze({ - role: 'tokenizerConfig', - name: 'tokenizer_config.json', - path: 'tokenizer_config.json', - size: 366, - sha256: '9261e7d79b44c8195c1cada2b453e55b00aeb81e907a6664974b4d7776172ab3' - }) - ]), - output: Object.freeze({ - name: 'last_hidden_state', - pooling: 'mean', - normalize: true - }) -}); - -async function createEmbeddingRuntime(model = DEFAULT_MODEL) { - validateModelDescriptor(model); - const modelDir = getModelCacheDir(getModelCacheRoot(), model); - await downloadModelIfNeeded(modelDir, model); +async function createEmbeddingRuntime(configuration, options = {}) { + const { model, modelDir } = await resolveEmbeddingModel(configuration, options); const { session, tokenizer } = await loadModelAndTokenizer(modelDir, model); const tokenizerState = createTokenizerState(tokenizer, model.maxLength); @@ -75,6 +41,81 @@ async function createEmbeddingRuntime(model = DEFAULT_MODEL) { return runtime; } +async function resolveEmbeddingModel(configuration, options = {}) { + const { root = process.cwd(), env = process.env } = options; + if (configuration?.directory) { + const modelDir = path.resolve(root, configuration.directory); + if (typeof configuration.model !== 'string' || !configuration.model) { + throw new Error('embedding.model is required when embedding.directory is configured'); + } + + let model; + try { + model = await readModelLock(modelDir); + } catch (error) { + const recovery = /Embedding model lock not found/.test(error.message) + ? modelInstallHint(configuration.model, configuration.directory) + : `Remove or replace the invalid lock explicitly, then ${lowercaseFirst( + modelInstallHint(configuration.model, configuration.directory) + )}`; + throw new Error(`${error.message}. ${recovery}`, { cause: error }); + } + + const preset = findModelPreset(configuration.model); + if (preset && modelDescriptorDigest(model) !== modelDescriptorDigest(preset)) { + throw new Error( + `Embedding model directory ${modelDir} does not contain the pinned ${configuration.model} preset. Choose another directory or remove it explicitly before provisioning it again.` + ); + } + if (!preset && model.repository !== configuration.model) { + throw new Error( + `Embedding model directory ${modelDir} contains ${model.repository}, not ${configuration.model}. Choose another directory or remove it explicitly before provisioning it again.` + ); + } + try { + await verifyModelDirectory(modelDir, model); + } catch (error) { + throw new Error( + `${error.message}. ${modelInstallHint(configuration.model, configuration.directory)}`, + { + cause: error + } + ); + } + return { model, modelDir }; + } + + const model = resolveConfiguredModel(configuration); + const modelDir = getModelCacheDir(getModelCacheRoot(env), model); + try { + await verifyModelDirectory(modelDir, model); + } catch (error) { + throw new Error(`${error.message}. ${modelInstallHint(model.repository)}`, { + cause: error + }); + } + return { model, modelDir }; +} + +function modelInstallHint(repository, directory) { + const modelArgument = findModelPreset(repository) + ? repository + : '--descriptor '; + const directoryArgument = directory ? ` --directory ${directory}` : ''; + return `Run 'npx cds-ai model install ${modelArgument}${directoryArgument}'.`; +} + +function lowercaseFirst(value) { + return `${value[0].toLowerCase()}${value.slice(1)}`; +} + +function resolveConfiguredModel(configuration) { + if (configuration == null) return DEFAULT_MODEL; + if (typeof configuration === 'string') return resolveModelPreset(configuration); + if (configuration.model) return resolveModelPreset(configuration.model); + return validateModelDescriptor(configuration); +} + function createTokenizerState(tokenizer, maxLength) { const probeText = 'embedding tokenizer boundary probe'; const content = normalizeEncoding( @@ -287,20 +328,6 @@ function validateAttentionMask(mask) { return mask; } -function getDataDir(appName = 'semantic-search') { - const home = os.homedir(); - const directory = - os.platform() === 'win32' - ? process.env.LOCALAPPDATA || process.env.APPDATA || path.join(home, 'AppData', 'Local') - : process.env.XDG_DATA_HOME || path.join(home, '.local', 'share'); - - return path.join(directory, appName); -} - -function getModelCacheRoot() { - return process.env.CDS_AI_MODEL_CACHE || path.join(getDataDir(), 'models'); -} - let defaultRuntime; let defaultInitialization; @@ -337,6 +364,7 @@ export { createTokenizerState, embedding, poolOutput, + resolveEmbeddingModel, tokenizeWithChunks }; diff --git a/lib/vector_embedding/model-utils.js b/lib/vector_embedding/model-utils.js index d3be040..5da2609 100644 --- a/lib/vector_embedding/model-utils.js +++ b/lib/vector_embedding/model-utils.js @@ -1,13 +1,19 @@ import { createHash, randomUUID } from 'crypto'; import { createReadStream } from 'fs'; import fs from 'fs/promises'; +import os from 'os'; import path from 'path'; -import { Tokenizer } from '@huggingface/tokenizers'; -import { InferenceSession } from './InferenceSession.js'; const DOWNLOAD_TIMEOUT_MS = 5 * 60 * 1000; +const MODEL_LOCK_FILE = 'embedding.lock.json'; +const MODEL_INSTALL_LOCK_FILE = '.embedding.install.lock'; +const MODEL_LOCK_VERSION = 1; +const INSTALL_LOCK_STALE_MS = 30 * 60 * 1000; +const PROVISIONED_DIRECTORY_MODE = 0o755; +const PROVISIONED_FILE_MODE = 0o644; const REQUIRED_FILE_ROLES = ['model', 'tokenizer', 'tokenizerConfig']; const ALLOWED_FILE_ROLES = new Set([...REQUIRED_FILE_ROLES, 'auxiliary']); +const RESERVED_ARTIFACT_PATHS = [MODEL_LOCK_FILE, MODEL_INSTALL_LOCK_FILE]; function validateModelDescriptor(model) { if (!model || typeof model !== 'object' || Array.isArray(model)) { @@ -44,6 +50,10 @@ function validateModelDescriptor(model) { assertSafeRelativePath(file.name, `embedding.files[${file.role}].name`); assertSafeRelativePath(file.path, `embedding.files[${file.role}].path`); if (names.has(file.name)) throw new Error(`Duplicate embedding file name '${file.name}'`); + for (const existingName of names) assertNoPathCollision(file.name, existingName); + for (const reservedPath of RESERVED_ARTIFACT_PATHS) { + assertNoPathCollision(file.name, reservedPath, 'provisioning metadata'); + } names.add(file.name); if (!Number.isSafeInteger(file.size) || file.size < 1) { throw new Error(`Invalid size for embedding file '${file.name}'`); @@ -73,6 +83,18 @@ function validateModelDescriptor(model) { return model; } +function assertNoPathCollision(left, right, description = 'another embedding file') { + const normalizedLeft = left.toLowerCase(); + const normalizedRight = right.toLowerCase(); + if ( + normalizedLeft === normalizedRight || + normalizedLeft.startsWith(`${normalizedRight}/`) || + normalizedRight.startsWith(`${normalizedLeft}/`) + ) { + throw new Error(`Embedding file '${left}' conflicts with ${description} '${right}'`); + } +} + function assertSafeRepository(repository) { if ( typeof repository !== 'string' || @@ -84,19 +106,26 @@ function assertSafeRepository(repository) { } function assertSafeRelativePath(value, field) { + const parts = typeof value === 'string' ? value.split('/') : []; if ( typeof value !== 'string' || value.length === 0 || value.includes('\\') || path.posix.isAbsolute(value) || path.posix.normalize(value) !== value || - value.split('/').some((part) => part === '' || part === '.' || part === '..') || + parts.some((part) => part === '' || part === '.' || part === '..') || + parts.some((part) => part.endsWith('.') || isWindowsDeviceName(part)) || !/^[A-Za-z0-9._/-]+$/.test(value) ) { throw new Error(`${field} must be a safe relative path`); } } +function isWindowsDeviceName(value) { + const basename = value.split('.')[0].toUpperCase(); + return /^(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])$/.test(basename); +} + function fileForRole(model, role) { return model.files.find((file) => file.role === role); } @@ -119,6 +148,32 @@ function artifactSetDigest(model) { return createHash('sha256').update(canonical).digest('hex'); } +function modelDescriptorDigest(model) { + validateModelDescriptor(model); + const files = model.files + .map(({ role, name, path: remotePath, size, sha256: checksum }) => ({ + role, + name, + path: remotePath, + size, + sha256: checksum + })) + .sort((left, right) => JSON.stringify(left).localeCompare(JSON.stringify(right))); + const canonical = JSON.stringify({ + repository: model.repository, + revision: model.revision.toLowerCase(), + dimensions: model.dimensions, + maxLength: model.maxLength, + files, + output: { + name: model.output.name, + pooling: model.output.pooling, + normalize: model.output.normalize + } + }); + return createHash('sha256').update(canonical).digest('hex'); +} + function getModelCacheDir(cacheRoot, model) { validateModelDescriptor(model); return path.join( @@ -129,6 +184,23 @@ function getModelCacheDir(cacheRoot, model) { ); } +function getDataDir(appName = 'semantic-search', environment = process.env) { + const home = os.homedir(); + const directory = + os.platform() === 'win32' + ? environment.LOCALAPPDATA || environment.APPDATA || path.join(home, 'AppData', 'Local') + : environment.XDG_DATA_HOME || path.join(home, '.local', 'share'); + + return path.join(directory, appName); +} + +function getModelCacheRoot(environment = process.env) { + return ( + environment.CDS_AI_MODEL_CACHE || + path.join(getDataDir('semantic-search', environment), 'models') + ); +} + async function sha256(filePath) { const hash = createHash('sha256'); for await (const chunk of createReadStream(filePath)) hash.update(chunk); @@ -137,7 +209,7 @@ async function sha256(filePath) { async function isValidFile(filePath, file) { try { - const stat = await fs.stat(filePath); + const stat = await fs.lstat(filePath); return stat.isFile() && stat.size === file.size && (await sha256(filePath)) === file.sha256; } catch (error) { if (error.code === 'ENOENT') return false; @@ -145,6 +217,27 @@ async function isValidFile(filePath, file) { } } +async function verifyModelDirectory(modelDir, model) { + validateModelDescriptor(model); + await assertModelDirectory(modelDir); + const validity = await Promise.all( + model.files.map(async (file) => { + await assertNoSymlinkComponents(modelDir, file.name); + return { + file, + valid: await isValidFile(path.join(modelDir, file.name), file) + }; + }) + ); + const invalid = validity.filter(({ valid }) => !valid).map(({ file }) => file.name); + if (invalid.length > 0) { + throw new Error( + `Embedding model is not provisioned or failed integrity checks in ${modelDir}. Missing or invalid files: ${invalid.join(', ')}` + ); + } + return modelDir; +} + async function downloadFile(url, outputPath, file, options = {}) { const { fetchImpl = globalThis.fetch, timeoutMs = DOWNLOAD_TIMEOUT_MS } = options; const controller = new AbortController(); @@ -167,7 +260,7 @@ async function downloadFile(url, outputPath, file, options = {}) { } await fs.mkdir(path.dirname(outputPath), { recursive: true }); - handle = await fs.open(temporaryPath, 'wx', 0o600); + handle = await fs.open(temporaryPath, 'wx', PROVISIONED_FILE_MODE); const hash = createHash('sha256'); let bytesWritten = 0; @@ -193,6 +286,8 @@ async function downloadFile(url, outputPath, file, options = {}) { throw new Error(`Invalid SHA-256 for ${url}: expected ${file.sha256}, received ${digest}`); } + await fs.chmod(temporaryPath, PROVISIONED_FILE_MODE); + try { await fs.rename(temporaryPath, outputPath); } catch (error) { @@ -203,6 +298,7 @@ async function downloadFile(url, outputPath, file, options = {}) { await fs.rename(temporaryPath, outputPath); } } + await fs.chmod(outputPath, PROVISIONED_FILE_MODE); } catch (error) { if (error.name === 'AbortError') { throw new Error(`Timed out after ${timeoutMs} ms while downloading ${url}`, { cause: error }); @@ -217,12 +313,19 @@ async function downloadFile(url, outputPath, file, options = {}) { async function downloadModelIfNeeded(modelDir, model, options) { validateModelDescriptor(model); - await fs.mkdir(modelDir, { recursive: true }); + await ensureDirectory(modelDir); for (const file of model.files) { const filePath = path.join(modelDir, file.name); // eslint-disable-next-line no-await-in-loop - if (await isValidFile(filePath, file)) continue; + await prepareArtifactPath(modelDir, file.name); + // eslint-disable-next-line no-await-in-loop + if (await isValidFile(filePath, file)) { + // Keep build-time provisioning readable when the runtime uses another UID. + // eslint-disable-next-line no-await-in-loop + await fs.chmod(filePath, PROVISIONED_FILE_MODE); + continue; + } const url = `https://huggingface.co/${model.repository}/resolve/${model.revision}/${file.path}`; // Files are downloaded serially to avoid multiplying startup bandwidth and memory usage. @@ -231,7 +334,412 @@ async function downloadModelIfNeeded(modelDir, model, options) { } } +async function prepareArtifactPath(modelDir, relativePath) { + await assertNoSymlinkComponents(modelDir, relativePath); + let current = modelDir; + for (const part of relativePath.split('/').slice(0, -1)) { + current = path.join(current, part); + // eslint-disable-next-line no-await-in-loop + await fs.mkdir(current, { mode: PROVISIONED_DIRECTORY_MODE }).catch((error) => { + if (error.code !== 'EEXIST') throw error; + }); + // eslint-disable-next-line no-await-in-loop + const stat = await fs.lstat(current); + if (stat.isSymbolicLink()) { + throw new Error(`Embedding artifact path must not contain symbolic links: ${current}`); + } + if (!stat.isDirectory()) { + throw new Error(`Embedding artifact parent is not a directory: ${current}`); + } + // eslint-disable-next-line no-await-in-loop + await fs.chmod(current, PROVISIONED_DIRECTORY_MODE); + } + await assertNoSymlinkComponents(modelDir, relativePath); +} + +async function assertNoSymlinkComponents(modelDir, relativePath) { + const exists = await assertModelDirectory(modelDir); + if (!exists) return; + let current = modelDir; + const parts = relativePath.split('/'); + for (let index = 0; index < parts.length; index++) { + current = path.join(current, parts[index]); + let stat; + try { + // eslint-disable-next-line no-await-in-loop + stat = await fs.lstat(current); + } catch (error) { + if (error.code === 'ENOENT') return; + throw error; + } + if (stat.isSymbolicLink()) { + throw new Error(`Embedding artifact path must not contain symbolic links: ${current}`); + } + if (index < parts.length - 1 && !stat.isDirectory()) { + throw new Error(`Embedding artifact parent is not a directory: ${current}`); + } + } +} + +async function assertModelDirectory(modelDir) { + try { + const stat = await fs.lstat(modelDir); + if (stat.isSymbolicLink()) { + throw new Error(`Embedding model directory must not be a symbolic link: ${modelDir}`); + } + if (!stat.isDirectory()) { + throw new Error(`Embedding model path is not a directory: ${modelDir}`); + } + return true; + } catch (error) { + if (error.code === 'ENOENT') return false; + throw error; + } +} + +async function ensureDirectory(directory) { + await ensureParentDirectories(directory); + await assertModelDirectory(directory); + await fs.chmod(directory, PROVISIONED_DIRECTORY_MODE); +} + +async function ensureParentDirectories(directory) { + const missing = []; + let current = directory; + // eslint-disable-next-line no-await-in-loop + while (!(await pathExists(current))) { + missing.push(current); + const parent = path.dirname(current); + if (parent === current) break; + current = parent; + } + + for (const missingDirectory of missing.reverse()) { + // eslint-disable-next-line no-await-in-loop + await fs.mkdir(missingDirectory, { mode: PROVISIONED_DIRECTORY_MODE }).catch((error) => { + if (error.code !== 'EEXIST') throw error; + }); + // Explicit chmod avoids umask making build-time model directories unreadable at runtime. + // eslint-disable-next-line no-await-in-loop + await fs.chmod(missingDirectory, PROVISIONED_DIRECTORY_MODE); + } +} + +async function pathExists(filePath) { + try { + await fs.lstat(filePath); + return true; + } catch (error) { + if (error.code === 'ENOENT') return false; + throw error; + } +} + +async function canonicalizeProvisioningDirectory(directory) { + const requestedDirectory = path.resolve(directory); + let current = requestedDirectory; + const missing = []; + + while (true) { + try { + // eslint-disable-next-line no-await-in-loop + const stat = await fs.lstat(current); + if (current === requestedDirectory && stat.isSymbolicLink()) { + throw new Error(`Embedding model directory must not be a symbolic link: ${directory}`); + } + // eslint-disable-next-line no-await-in-loop + const canonicalAncestor = await fs.realpath(current); + // eslint-disable-next-line no-await-in-loop + const canonicalStat = await fs.stat(canonicalAncestor); + if (!canonicalStat.isDirectory()) { + throw new Error(`Embedding model path is not a directory: ${current}`); + } + return path.join(canonicalAncestor, ...missing.reverse()); + } catch (error) { + if (error.code !== 'ENOENT') throw error; + missing.push(path.basename(current)); + const parent = path.dirname(current); + if (parent === current) throw error; + current = parent; + } + } +} + +async function readModelLock(modelDir) { + const lockPath = path.join(modelDir, MODEL_LOCK_FILE); + let lock; + try { + await assertModelDirectory(modelDir); + const stat = await fs.lstat(lockPath); + if (stat.isSymbolicLink()) { + throw new Error(`Embedding model lock must not be a symbolic link at ${lockPath}`); + } + lock = JSON.parse(await fs.readFile(lockPath, 'utf8')); + } catch (error) { + if (error.code === 'ENOENT') { + throw new Error(`Embedding model lock not found at ${lockPath}`, { cause: error }); + } + throw new Error(`Cannot read embedding model lock at ${lockPath}: ${error.message}`, { + cause: error + }); + } + if (lock.formatVersion !== MODEL_LOCK_VERSION) { + throw new Error( + `Unsupported embedding model lock version ${lock.formatVersion ?? 'missing'} at ${lockPath}` + ); + } + const { formatVersion, ...model } = lock; + void formatVersion; + return validateModelDescriptor(model); +} + +async function writeModelLock(modelDir, model) { + const lockPath = path.join(modelDir, MODEL_LOCK_FILE); + const temporaryPath = `${lockPath}.${process.pid}.${randomUUID()}.tmp`; + const contents = `${JSON.stringify({ ...model, formatVersion: MODEL_LOCK_VERSION }, null, 2)}\n`; + let handle; + try { + await ensureDirectory(modelDir); + handle = await fs.open(temporaryPath, 'wx', PROVISIONED_FILE_MODE); + await handle.writeFile(contents); + await handle.sync(); + await handle.close(); + handle = undefined; + await fs.chmod(temporaryPath, PROVISIONED_FILE_MODE); + await fs.rename(temporaryPath, lockPath); + await fs.chmod(lockPath, PROVISIONED_FILE_MODE); + } finally { + await handle?.close().catch(() => {}); + await fs.unlink(temporaryPath).catch(() => {}); + } +} + +async function provisionModel(model, options = {}) { + validateModelDescriptor(model); + const requestedDirectory = path.resolve( + options.directory ?? getModelCacheDir(getModelCacheRoot(options.env), model) + ); + const directory = await canonicalizeProvisioningDirectory(requestedDirectory); + await ensureParentDirectories(path.dirname(directory)); + + return withInstallLock(directory, async () => { + const directoryExists = await assertModelDirectory(directory); + let lockedModel; + try { + lockedModel = await readModelLock(directory); + if (modelDescriptorDigest(lockedModel) !== modelDescriptorDigest(model)) { + throw new Error( + `Embedding model directory ${directory} is locked to a different model descriptor for ${lockedModel.repository}@${lockedModel.revision}. Choose another directory or remove it explicitly.` + ); + } + } catch (error) { + if (!/Embedding model lock not found/.test(error.message)) throw error; + } + + if (directoryExists) { + try { + await verifyModelDirectory(directory, model); + if (!lockedModel) await writeModelLock(directory, model); + await makeModelDirectoryReadable(directory, model); + return directory; + } catch (error) { + if (/symbolic link/.test(error.message)) throw error; + if (!lockedModel && (await directoryHasEntries(directory))) { + throw new Error( + `Embedding model directory ${directory} is not empty and has no valid lock. Choose an empty directory or remove its contents explicitly.`, + { cause: error } + ); + } + } + } + + const stagingDirectory = await createStagingDirectory(directory); + let published = false; + try { + await downloadModelIfNeeded(stagingDirectory, model, options); + await verifyModelDirectory(stagingDirectory, model); + await writeModelLock(stagingDirectory, model); + await publishModelDirectory(stagingDirectory, directory); + published = true; + } finally { + if (!published) await fs.rm(stagingDirectory, { recursive: true, force: true }); + } + return directory; + }); +} + +async function withInstallLock(directory, callback) { + const lockPath = path.join( + path.dirname(directory), + `.${path.basename(directory)}${MODEL_INSTALL_LOCK_FILE}` + ); + const owner = { + formatVersion: 1, + pid: process.pid, + hostname: os.hostname(), + createdAt: new Date().toISOString(), + token: randomUUID() + }; + let handle; + let heartbeat; + try { + handle = await acquireInstallLock(lockPath, directory, owner); + heartbeat = setInterval( + () => { + const now = new Date(); + fs.utimes(lockPath, now, now).catch(() => {}); + }, + Math.min(INSTALL_LOCK_STALE_MS / 3, 60 * 1000) + ); + heartbeat.unref(); + return await callback(); + } finally { + clearInterval(heartbeat); + await handle?.close().catch(() => {}); + if (handle) await releaseInstallLock(lockPath, owner); + } +} + +async function acquireInstallLock(lockPath, directory, owner) { + try { + return await createInstallLock(lockPath, owner); + } catch (error) { + if (error.code !== 'EEXIST') throw error; + if (await recoverStaleInstallLock(lockPath)) { + return createInstallLock(lockPath, owner); + } + throw new Error(`Embedding model directory ${directory} is already being provisioned`, { + cause: error + }); + } +} + +async function createInstallLock(lockPath, owner) { + let handle; + try { + handle = await fs.open(lockPath, 'wx', 0o600); + await handle.writeFile(`${JSON.stringify(owner)}\n`); + await handle.sync(); + return handle; + } catch (error) { + await handle?.close().catch(() => {}); + if (handle) await fs.unlink(lockPath).catch(() => {}); + throw error; + } +} + +async function recoverStaleInstallLock(lockPath) { + let contents; + let stat; + try { + [contents, stat] = await Promise.all([fs.readFile(lockPath, 'utf8'), fs.lstat(lockPath)]); + } catch (error) { + return error.code === 'ENOENT'; + } + + let owner; + try { + owner = JSON.parse(contents); + } catch { + if (Date.now() - stat.mtimeMs <= INSTALL_LOCK_STALE_MS) return false; + } + if (!isStaleInstallLock(owner, stat)) return false; + + const stalePath = `${lockPath}.${randomUUID()}.stale`; + try { + await fs.rename(lockPath, stalePath); + const movedContents = await fs.readFile(stalePath, 'utf8'); + if (movedContents !== contents) { + await fs.rename(stalePath, lockPath).catch(() => {}); + return false; + } + await fs.unlink(stalePath); + return true; + } catch (error) { + await fs.unlink(stalePath).catch(() => {}); + return error.code === 'ENOENT'; + } +} + +function isStaleInstallLock(owner, stat) { + if (!owner || typeof owner !== 'object') { + return Date.now() - stat.mtimeMs > INSTALL_LOCK_STALE_MS; + } + if (owner.hostname === os.hostname() && Number.isSafeInteger(owner.pid)) { + try { + process.kill(owner.pid, 0); + return false; + } catch (error) { + if (error.code === 'EPERM') return false; + if (error.code === 'ESRCH') return true; + return false; + } + } + return Date.now() - stat.mtimeMs > INSTALL_LOCK_STALE_MS; +} + +async function releaseInstallLock(lockPath, owner) { + try { + const currentOwner = JSON.parse(await fs.readFile(lockPath, 'utf8')); + if (currentOwner.token === owner.token) await fs.unlink(lockPath); + } catch (error) { + if (error.code !== 'ENOENT') throw error; + } +} + +async function createStagingDirectory(directory) { + const parent = path.dirname(directory); + const stagingDirectory = await fs.mkdtemp( + path.join(parent, `.${path.basename(directory)}.staging-`) + ); + await fs.chmod(stagingDirectory, PROVISIONED_DIRECTORY_MODE); + return stagingDirectory; +} + +async function publishModelDirectory(stagingDirectory, directory) { + const targetExists = await assertModelDirectory(directory); + let backupDirectory; + if (targetExists) { + backupDirectory = path.join( + path.dirname(directory), + `.${path.basename(directory)}.${randomUUID()}.backup` + ); + await fs.rename(directory, backupDirectory); + } + + try { + await fs.rename(stagingDirectory, directory); + } catch (error) { + if (backupDirectory) await fs.rename(backupDirectory, directory).catch(() => {}); + throw error; + } + if (backupDirectory) await fs.rm(backupDirectory, { recursive: true, force: true }); +} + +async function directoryHasEntries(directory) { + return (await fs.readdir(directory)).length > 0; +} + +async function makeModelDirectoryReadable(directory, model) { + await fs.chmod(directory, PROVISIONED_DIRECTORY_MODE); + for (const file of model.files) { + let current = directory; + for (const part of file.name.split('/').slice(0, -1)) { + current = path.join(current, part); + // eslint-disable-next-line no-await-in-loop + await fs.chmod(current, PROVISIONED_DIRECTORY_MODE); + } + // eslint-disable-next-line no-await-in-loop + await fs.chmod(path.join(directory, file.name), PROVISIONED_FILE_MODE); + } + await fs.chmod(path.join(directory, MODEL_LOCK_FILE), PROVISIONED_FILE_MODE); +} + async function loadModelAndTokenizer(modelDir, model) { + const [{ Tokenizer }, { InferenceSession }] = await Promise.all([ + import('@huggingface/tokenizers'), + import('./InferenceSession.js') + ]); const modelPath = path.join(modelDir, fileForRole(model, 'model').name); const tokenizerPath = path.join(modelDir, fileForRole(model, 'tokenizer').name); const tokenizerConfigPath = path.join(modelDir, fileForRole(model, 'tokenizerConfig').name); @@ -247,12 +755,18 @@ async function loadModelAndTokenizer(modelDir, model) { } export { + MODEL_LOCK_FILE, downloadFile, downloadModelIfNeeded, artifactSetDigest, fileForRole, getModelCacheDir, + getModelCacheRoot, isValidFile, loadModelAndTokenizer, + modelDescriptorDigest, + provisionModel, + readModelLock, + verifyModelDirectory, validateModelDescriptor }; diff --git a/lib/vector_embedding/models.js b/lib/vector_embedding/models.js new file mode 100644 index 0000000..c3c08b3 --- /dev/null +++ b/lib/vector_embedding/models.js @@ -0,0 +1,52 @@ +const DEFAULT_MODEL = Object.freeze({ + repository: 'Xenova/all-MiniLM-L6-v2', + revision: '751bff37182d3f1213fa05d7196b954e230abad9', + dimensions: 384, + maxLength: 128, + files: Object.freeze([ + Object.freeze({ + role: 'model', + name: 'model.onnx', + path: 'onnx/model.onnx', + size: 90387606, + sha256: '759c3cd2b7fe7e93933ad23c4c9181b7396442a2ed746ec7c1d46192c469c46e' + }), + Object.freeze({ + role: 'tokenizer', + name: 'tokenizer.json', + path: 'tokenizer.json', + size: 711661, + sha256: 'da0e79933b9ed51798a3ae27893d3c5fa4a201126cef75586296df9b4d2c62a0' + }), + Object.freeze({ + role: 'tokenizerConfig', + name: 'tokenizer_config.json', + path: 'tokenizer_config.json', + size: 366, + sha256: '9261e7d79b44c8195c1cada2b453e55b00aeb81e907a6664974b4d7776172ab3' + }) + ]), + output: Object.freeze({ + name: 'last_hidden_state', + pooling: 'mean', + normalize: true + }) +}); + +const MODEL_PRESETS = new Map([[DEFAULT_MODEL.repository, DEFAULT_MODEL]]); + +function findModelPreset(name) { + return MODEL_PRESETS.get(name); +} + +function resolveModelPreset(name) { + const model = findModelPreset(name); + if (!model) { + throw new Error( + `Unsupported embedding model '${name}'. Use --descriptor for a compatible custom model.` + ); + } + return model; +} + +export { DEFAULT_MODEL, findModelPreset, resolveModelPreset }; diff --git a/package.json b/package.json index c696aaa..16b00e2 100644 --- a/package.json +++ b/package.json @@ -8,8 +8,14 @@ "author": "SAP SE (https://www.sap.com)", "homepage": "https://cap.cloud.sap/", "main": "cds-plugin.js", + "bin": { + "cds-ai": "bin/cds-ai.js" + }, "scripts": { "lint": "npx -y eslint@10 .", + "model:provision": "node bin/cds-ai.js model install Xenova/all-MiniLM-L6-v2", + "pretest": "npm run model:provision", + "pretest:hybrid": "npm run model:provision", "test": "node --test tests/*.test.js", "test:hybrid": "cds bind --exec -- node --test tests/*.test.js tests/integration/*.test.js", "format": "npx -y prettier@3 . --write && format-cds -f", @@ -17,6 +23,7 @@ }, "files": [ "CHANGELOG.md", + "bin", "lib", "srv" ], diff --git a/tests/model-provisioning.test.js b/tests/model-provisioning.test.js new file mode 100644 index 0000000..095858d --- /dev/null +++ b/tests/model-provisioning.test.js @@ -0,0 +1,425 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, test } from 'node:test'; +import { runModelCommand } from '../lib/vector_embedding/cli.js'; +import { resolveEmbeddingModel } from '../lib/vector_embedding/embedding.js'; +import { + MODEL_LOCK_FILE, + provisionModel, + readModelLock, + verifyModelDirectory +} from '../lib/vector_embedding/model-utils.js'; +import { DEFAULT_MODEL, resolveModelPreset } from '../lib/vector_embedding/models.js'; + +const temporaryDirectories = []; + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => fs.rm(directory, { recursive: true, force: true })) + ); +}); + +describe('model presets', () => { + test('resolves the default MiniLM model by repository name', () => { + assert.equal(resolveModelPreset('Xenova/all-MiniLM-L6-v2'), DEFAULT_MODEL); + }); + + test('rejects unknown model names', () => { + assert.throws(() => resolveModelPreset('example/unknown'), /Unsupported embedding model/); + }); +}); + +describe('explicit model provisioning', () => { + test('downloads, verifies, and locks a model idempotently', async () => { + const directory = await createTemporaryDirectory(); + const content = Buffer.from('verified model fixture'); + const model = fixtureModel(content); + const requestedUrls = []; + const fetchImpl = createFetch(content, requestedUrls); + + await provisionModel(model, { directory, fetchImpl }); + await provisionModel(model, { directory, fetchImpl }); + + assert.deepEqual( + requestedUrls, + model.files.map( + (file) => `https://huggingface.co/example/model/resolve/${model.revision}/${file.path}` + ) + ); + assert.deepEqual(await readModelLock(directory), model); + assert.deepEqual((await fs.readdir(directory)).sort(), [ + MODEL_LOCK_FILE, + 'model.onnx', + 'tokenizer.json', + 'tokenizer_config.json' + ]); + const modes = await Promise.all( + [MODEL_LOCK_FILE, ...model.files.map(({ name }) => name)].map(async (file) => + fs.stat(path.join(directory, file)).then(({ mode }) => mode & 0o777) + ) + ); + assert.deepEqual(modes, new Array(modes.length).fill(0o644)); + }); + + test('restores readable permissions on already valid artifacts', async () => { + const directory = await createTemporaryDirectory(); + const content = Buffer.from('readable model fixture'); + const model = fixtureModel(content); + const modelPath = path.join(directory, model.files[0].name); + await provisionModel(model, { directory, fetchImpl: createFetch(content) }); + await fs.chmod(modelPath, 0o600); + + await provisionModel(model, { directory, fetchImpl: createFetch(content) }); + + assert.equal((await fs.stat(modelPath)).mode & 0o777, 0o644); + }); + + test('makes newly provisioned directories traversable across runtime users', async () => { + const parent = await createTemporaryDirectory(); + const modelsDirectory = path.join(parent, 'models'); + const directory = path.join(modelsDirectory, 'custom'); + const content = Buffer.from('directory mode fixture'); + const baseModel = fixtureModel(content); + const model = { + ...baseModel, + files: baseModel.files.map((file, index) => + index === 0 ? { ...file, name: 'onnx/model.onnx' } : file + ) + }; + const originalUmask = process.umask(0o077); + try { + await provisionModel(model, { directory, fetchImpl: createFetch(content) }); + } finally { + process.umask(originalUmask); + } + + assert.equal((await fs.stat(modelsDirectory)).mode & 0o777, 0o755); + assert.equal((await fs.stat(directory)).mode & 0o777, 0o755); + assert.equal((await fs.stat(path.join(directory, 'onnx'))).mode & 0o777, 0o755); + assert.equal((await fs.stat(path.join(directory, 'onnx/model.onnx'))).mode & 0o777, 0o644); + assert.equal((await fs.stat(path.join(directory, MODEL_LOCK_FILE))).mode & 0o777, 0o644); + }); + + test('does not repurpose a directory locked to a different descriptor', async () => { + const directory = await createTemporaryDirectory(); + const content = Buffer.from('locked model fixture'); + const model = fixtureModel(content); + await provisionModel(model, { directory, fetchImpl: createFetch(content) }); + + await assert.rejects( + provisionModel({ ...model, repository: 'example/other-model' }, { directory }), + /locked to a different model descriptor/ + ); + await assert.rejects( + provisionModel({ ...model, dimensions: model.dimensions + 1 }, { directory }), + /locked to a different model descriptor/ + ); + assert.deepEqual(await readModelLock(directory), model); + }); + + test('serializes provisioning attempts for the same directory', async () => { + const directory = await createTemporaryDirectory(); + const content = Buffer.from('concurrent model fixture'); + const model = fixtureModel(content); + let releaseDownload; + let signalDownloadStarted; + const downloadStarted = new Promise((resolve) => { + signalDownloadStarted = resolve; + }); + const waitForRelease = new Promise((resolve) => { + releaseDownload = resolve; + }); + let firstRequest = true; + const fetchImpl = async () => { + if (firstRequest) { + firstRequest = false; + signalDownloadStarted(); + await waitForRelease; + } + return new Response(content); + }; + + const first = provisionModel(model, { directory, fetchImpl }); + await downloadStarted; + await assert.rejects( + provisionModel(model, { directory, fetchImpl }), + /already being provisioned/ + ); + releaseDownload(); + await first; + }); + + test('recovers a stale install lock owned by a terminated local process', async () => { + const parent = await createTemporaryDirectory(); + const directory = path.join(parent, 'model'); + const installLock = path.join(parent, '.model.embedding.install.lock'); + const content = Buffer.from('stale lock fixture'); + const model = fixtureModel(content); + await fs.writeFile( + installLock, + JSON.stringify({ + formatVersion: 1, + pid: 99999999, + hostname: os.hostname(), + createdAt: '2000-01-01T00:00:00.000Z', + token: 'stale-owner' + }) + ); + + await provisionModel(model, { directory, fetchImpl: createFetch(content) }); + + await assert.rejects(fs.access(installLock)); + assert.deepEqual(await readModelLock(directory), model); + }); + + test('recovers a stale install lock truncated by an interrupted write', async () => { + const parent = await createTemporaryDirectory(); + const directory = path.join(parent, 'model'); + const installLock = path.join(parent, '.model.embedding.install.lock'); + const content = Buffer.from('truncated lock fixture'); + const model = fixtureModel(content); + await fs.writeFile(installLock, '{'); + const staleTime = new Date('2000-01-01T00:00:00.000Z'); + await fs.utimes(installLock, staleTime, staleTime); + + await provisionModel(model, { directory, fetchImpl: createFetch(content) }); + + await assert.rejects(fs.access(installLock)); + assert.deepEqual(await readModelLock(directory), model); + }); + + test('rejects symlinked artifact path components', async () => { + const directory = await createTemporaryDirectory(); + const outside = await createTemporaryDirectory(); + const content = Buffer.from('symlink model fixture'); + const baseModel = fixtureModel(content); + const model = { + ...baseModel, + files: baseModel.files.map((file, index) => + index === 0 ? { ...file, name: 'nested/model.onnx' } : file + ) + }; + await fs.symlink(outside, path.join(directory, 'nested'), 'dir'); + + await assert.rejects( + provisionModel(model, { directory, fetchImpl: createFetch(content) }), + /must not contain symbolic links/ + ); + assert.deepEqual(await fs.readdir(outside), []); + }); + + test('rejects a symlinked or replaced model directory', async () => { + const parent = await createTemporaryDirectory(); + const directory = path.join(parent, 'model'); + const outside = await createTemporaryDirectory(); + const content = Buffer.from('root symlink fixture'); + const model = fixtureModel(content); + await fs.symlink(outside, directory, 'dir'); + + await assert.rejects( + provisionModel(model, { directory, fetchImpl: createFetch(content) }), + /model directory must not be a symbolic link/ + ); + assert.deepEqual(await fs.readdir(outside), []); + }); + + test('canonicalizes symlinked ancestor directories before provisioning', async () => { + const parent = await createTemporaryDirectory(); + const outside = await createTemporaryDirectory(); + const modelsDirectory = path.join(parent, 'models'); + const requestedDirectory = path.join(modelsDirectory, 'custom'); + const content = Buffer.from('ancestor symlink fixture'); + const model = fixtureModel(content); + await fs.symlink(outside, modelsDirectory, 'dir'); + + const directory = await provisionModel(model, { + directory: requestedDirectory, + fetchImpl: createFetch(content) + }); + + assert.equal(directory, path.join(await fs.realpath(outside), 'custom')); + assert.deepEqual(await readModelLock(directory), model); + }); + + test('publishes from staging and detects replacement of the target during download', async () => { + const parent = await createTemporaryDirectory(); + const directory = path.join(parent, 'model'); + const outside = await createTemporaryDirectory(); + const content = Buffer.from('target replacement fixture'); + const model = fixtureModel(content); + let replaced = false; + const fetchImpl = async () => { + if (!replaced) { + replaced = true; + await fs.symlink(outside, directory, 'dir'); + } + return new Response(content); + }; + + await assert.rejects( + provisionModel(model, { directory, fetchImpl }), + /model directory must not be a symbolic link/ + ); + assert.deepEqual(await fs.readdir(outside), []); + }); + + test('writes the lock only after every artifact passes verification', async () => { + const directory = await createTemporaryDirectory(); + const content = Buffer.from('expected model fixture'); + const model = fixtureModel(content); + + await assert.rejects( + provisionModel(model, { + directory, + fetchImpl: async () => new Response(Buffer.from('invalid')) + }), + /Invalid size|Invalid SHA-256/ + ); + await assert.rejects(fs.access(path.join(directory, MODEL_LOCK_FILE))); + }); + + test('fails verification instead of downloading missing runtime files', async () => { + const directory = await createTemporaryDirectory(); + const model = fixtureModel(Buffer.from('fixture')); + + await assert.rejects( + verifyModelDirectory(directory, model), + new RegExp(`Embedding model is not provisioned.*${escapeRegExp(directory)}`, 's') + ); + assert.deepEqual(await fs.readdir(directory), []); + }); + + test('keeps runtime initialization offline', async () => { + const root = await createTemporaryDirectory(); + let fetched = false; + const originalFetch = globalThis.fetch; + globalThis.fetch = () => { + fetched = true; + throw new Error('runtime must not fetch'); + }; + + try { + await assert.rejects( + resolveEmbeddingModel( + { + model: DEFAULT_MODEL.repository, + directory: './models/minilm' + }, + { root } + ), + /cds-ai model install Xenova\/all-MiniLM-L6-v2 --directory \.\/models\/minilm/ + ); + assert.equal(fetched, false); + } finally { + globalThis.fetch = originalFetch; + } + }); + + test('requires the complete built-in preset in an explicitly configured directory', async () => { + const directory = await createTemporaryDirectory(); + const content = Buffer.from('imposter preset fixture'); + const model = { + ...fixtureModel(content), + repository: DEFAULT_MODEL.repository + }; + await provisionModel(model, { directory, fetchImpl: createFetch(content) }); + + await assert.rejects( + resolveEmbeddingModel({ model: DEFAULT_MODEL.repository, directory }), + /does not contain the pinned Xenova\/all-MiniLM-L6-v2 preset/ + ); + }); + + test('gives custom models a descriptor-based provisioning command', async () => { + const root = await createTemporaryDirectory(); + + await assert.rejects( + resolveEmbeddingModel({ model: 'example/custom', directory: './models/custom' }, { root }), + /model install --descriptor --directory \.\/models\/custom/ + ); + }); + + test('requires explicit lock recovery before reinstalling', async () => { + const directory = await createTemporaryDirectory(); + await fs.writeFile(path.join(directory, MODEL_LOCK_FILE), '{}'); + + await assert.rejects( + resolveEmbeddingModel({ model: DEFAULT_MODEL.repository, directory }), + /Remove or replace the invalid lock explicitly, then run 'npx cds-ai model install/ + ); + }); + + test('installs a descriptor through the command API', async () => { + const root = await createTemporaryDirectory(); + const directory = path.join(root, 'models', 'custom'); + const descriptor = path.join(root, 'embedding-model.json'); + const content = Buffer.from('command fixture'); + const model = fixtureModel(content); + const output = []; + + await fs.writeFile(descriptor, JSON.stringify(model)); + await runModelCommand( + ['model', 'install', '--descriptor', descriptor, '--directory', directory], + { + cwd: root, + fetchImpl: createFetch(content), + stdout: { write: (value) => output.push(value) } + } + ); + + assert.deepEqual(await readModelLock(directory), model); + assert.match(output.join(''), /Provisioned example\/model/); + }); +}); + +function createFetch(content, requestedUrls = []) { + return async (url) => { + requestedUrls.push(url); + return new Response(content, { + headers: { 'content-length': String(content.length) } + }); + }; +} + +async function createTemporaryDirectory() { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'cap-ai-provision-')); + temporaryDirectories.push(directory); + return directory; +} + +function fixtureModel(content) { + const sha256 = createHash('sha256').update(content).digest('hex'); + return { + repository: 'example/model', + revision: 'deadbeefdeadbeefdeadbeefdeadbeefdeadbeef', + dimensions: 2, + maxLength: 8, + files: [ + { role: 'model', name: 'model.onnx', path: 'onnx/model.onnx', size: content.length, sha256 }, + { + role: 'tokenizer', + name: 'tokenizer.json', + path: 'tokenizer.json', + size: content.length, + sha256 + }, + { + role: 'tokenizerConfig', + name: 'tokenizer_config.json', + path: 'tokenizer_config.json', + size: content.length, + sha256 + } + ], + output: { name: 'last_hidden_state', pooling: 'mean', normalize: true } + }; +} + +function escapeRegExp(value) { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/tests/vector-unit.test.js b/tests/vector-unit.test.js index 7162669..bb6f6d6 100644 --- a/tests/vector-unit.test.js +++ b/tests/vector-unit.test.js @@ -132,6 +132,38 @@ describe('model compatibility', () => { }), /safe relative path/ ); + assert.throws( + () => + validateModelDescriptor({ + ...model, + files: model.files.map((file, index) => + index === 0 ? { ...file, name: 'embedding.lock.json' } : file + ) + }), + /conflicts with provisioning metadata/ + ); + assert.throws( + () => + validateModelDescriptor({ + ...model, + files: model.files.map((file, index) => + index === 0 ? { ...file, name: 'EMBEDDING.LOCK.JSON' } : file + ) + }), + /conflicts with provisioning metadata/ + ); + assert.throws( + () => + validateModelDescriptor({ + ...model, + files: model.files.map((file, index) => { + if (index === 0) return { ...file, name: 'nested' }; + if (index === 1) return { ...file, name: 'nested/tokenizer.json' }; + return file; + }) + }), + /conflicts with another embedding file/ + ); }); }); diff --git a/tests/vector.test.js b/tests/vector.test.js index 50b3fe2..1ac3ff8 100644 --- a/tests/vector.test.js +++ b/tests/vector.test.js @@ -3,6 +3,7 @@ import assert from 'node:assert'; import cds from '@sap/cds'; import { initializeEmbedding, vector_embedding } from '../lib/vector_embedding/index.js'; import { DEFAULT_MODEL } from '../lib/vector_embedding/embedding.js'; +import { getModelCacheDir, getModelCacheRoot } from '../lib/vector_embedding/model-utils.js'; let embeddingModule; @@ -132,6 +133,10 @@ describe('ai-sqlite integration', () => { before(async () => { db = await cds.connect.to('vector-db', { kind: 'ai-sqlite', + embedding: { + model: DEFAULT_MODEL.repository, + directory: getModelCacheDir(getModelCacheRoot(), DEFAULT_MODEL) + }, credentials: { url: ':memory:' } }); }); From c626d7fe96216b4b0e2140cb39fdb9b7fbf24221 Mon Sep 17 00:00:00 2001 From: Sebastian Van Syckel Date: Wed, 26 Aug 2026 18:55:55 +0200 Subject: [PATCH 2/6] feat: support lazy embedding model provisioning --- CHANGELOG.md | 4 +- README.md | 38 ++++--- lib/sqlite/AISQLiteService.js | 3 +- lib/vector_embedding/cli.js | 5 +- lib/vector_embedding/embedding.js | 130 ++++++++++++++++------ lib/vector_embedding/model-utils.js | 11 +- lib/vector_embedding/models.js | 2 +- tests/model-provisioning.test.js | 160 ++++++++++++++++++++++++---- tests/vector.test.js | 39 +------ 9 files changed, 282 insertions(+), 110 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 793d0e9..cebda4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,9 @@ ### Added - Add the `ai-sqlite` kind with a `VECTOR_EMBEDDING` function using ONNX Runtime and the `Xenova/all-MiniLM-L6-v2` model (384 dimensions) - - Adds `cds-ai model install` for explicit, checksum-verified model provisioning; application startup remains offline + - Adds `cds-ai model install` for explicit, checksum-verified model provisioning and a warned, on-demand download into the default cache - Uses `@huggingface/tokenizers` and chunks long input without dropping per-chunk special tokens - - Allows an explicit, checksum-verified compatible encoder model descriptor per `ai-sqlite` service + - Configures embedding runtimes only through `model` and an optional relative or absolute `directory`; custom model metadata remains in the provisioned lock - Supports both 3-parameter `(text, text_type, model_and_version)` and 4-parameter variants with `remote_source` - Compatible with `SAP_GXY.20250407` and `SAP_GXY.20240715` model versions - Synchronous execution suitable for SQLite user-defined functions diff --git a/README.md b/README.md index 8251935..5ddecd2 100644 --- a/README.md +++ b/README.md @@ -219,26 +219,33 @@ npm add @cap-js/sqlite onnxruntime-node@1.20.1 `ai-sqlite` currently requires exactly `onnxruntime-node` 1.20.1 because synchronous SQLite functions need a version-specific native runtime API. -Provision the default model before starting the application: - -```sh -npx cds-ai model install Xenova/all-MiniLM-L6-v2 -``` - -The command downloads the pinned artifacts, verifies their sizes and SHA-256 checksums, and writes an `embedding.lock.json`. Application startup never downloads model files. - -Select `ai-sqlite` for the database service: +Select `ai-sqlite` and the embedding model by name: ```json { "cds": { "requires": { - "db": "ai-sqlite" + "db": { + "kind": "ai-sqlite", + "embedding": { + "model": "Xenova/all-MiniLM-L6-v2" + } + } } } } ``` +`Xenova/all-MiniLM-L6-v2` is used when the complete `embedding` option is omitted. If `embedding.directory` is not configured, startup checks the default model cache. A missing model is downloaded there after printing a warning, and subsequent starts reuse the verified files. + +To avoid a download during application startup, provision the model explicitly first: + +```sh +npx cds-ai model install Xenova/all-MiniLM-L6-v2 +``` + +The command uses the same default cache location as the runtime, verifies artifact sizes and SHA-256 checksums, and writes an `embedding.lock.json`. + The HANA-compatible SQL function can then be used in CQL: ```js @@ -260,7 +267,7 @@ SELECT.from('Books').columns` **Features:** - **Initialization**: The ONNX model is loaded when the `ai-sqlite` service starts -- **Explicit provisioning**: Model artifacts are downloaded only by `cds-ai model install`; runtime initialization is offline +- **Flexible provisioning**: Preinstall models with `cds-ai model install`, or let development startup download a missing built-in model after warning you - **Verified cache**: The pinned model revision and artifact set are stored by default below the user's data directory; set `CDS_AI_MODEL_CACHE` to select another cache root - **Hugging Face tokenization**: Uses `@huggingface/tokenizers` and safely chunks text that exceeds the model limit - **Deterministic**: Same input always produces same output @@ -314,7 +321,7 @@ Provision it into an application-managed directory: npx cds-ai model install --descriptor ./embedding-model.json --directory ./models/custom ``` -Then point the service at that locked directory: +Then point the service at that locked directory. Runtime configuration contains only the model name and directory; the lock contains the technical model metadata: ```json { @@ -332,14 +339,17 @@ Then point the service at that locked directory: } ``` -Relative directories are resolved from the CAP project root. Provisioning canonicalizes symlinked parent directories and rejects a model directory that is itself a symlink. For production, provision the directory while building the application image or mount it read-only. The inline descriptor form from earlier versions remains supported and uses the configured cache root. +Relative directories are resolved from `cds.root`. Absolute directories are used unchanged, which allows multiple applications to reuse a shared model directory. When `embedding.directory` is configured, startup never downloads or modifies that directory: provision it while building the application image or mount an already provisioned directory. Provisioning canonicalizes symlinked parent directories and rejects a model directory that is itself a symlink. + +The runtime accepts no model metadata beyond `embedding.model` and `embedding.directory`. Ad-hoc downloads by model name are available only for built-in presets. Custom models must first be installed from a descriptor into an explicitly configured directory. Compatible models must accept `input_ids` and may additionally accept `attention_mask` and `token_type_ids`, all as `int64` tensors. Their configured float32 or float64 output must support `mean` or `cls` pooling from `[1, sequence, dimensions]`, or `none` for an already pooled `[dimensions]` or `[1, dimensions]` tensor. Additional pinned ONNX data files can use the `auxiliary` role. Startup probes the model and rejects incompatible input names, output names, types, shapes, or dimensions. **Error Handling:** - Starting `ai-sqlite` fails if the ONNX model cannot be initialized -- Starting `ai-sqlite` fails with a provisioning command if model artifacts are missing or fail integrity checks +- A missing built-in model in the default cache is downloaded after a startup warning +- Starting `ai-sqlite` fails with a provisioning command if a configured model directory is missing or fails integrity checks - Provisioning downloads are time-limited and accepted only when their expected size and SHA-256 match - Throws if embedding generation fails diff --git a/lib/sqlite/AISQLiteService.js b/lib/sqlite/AISQLiteService.js index 07212ac..7473f47 100644 --- a/lib/sqlite/AISQLiteService.js +++ b/lib/sqlite/AISQLiteService.js @@ -7,7 +7,8 @@ const LOG = cds.log('@cap-js/ai'); export default class AISQLiteService extends SQLiteService { async init() { this._embeddingRuntime = await createEmbeddingRuntime(this.options.embedding, { - root: cds.root + root: cds.root, + warn: (message) => LOG.warn(message) }); LOG.info('Vector embedding ONNX model initialized'); return super.init(); diff --git a/lib/vector_embedding/cli.js b/lib/vector_embedding/cli.js index ea889c9..07f2543 100644 --- a/lib/vector_embedding/cli.js +++ b/lib/vector_embedding/cli.js @@ -10,7 +10,7 @@ import { resolveModelPreset } from './models.js'; const HELP = `Usage: cds-ai model install [--directory ] - cds-ai model install --descriptor [--directory ] + cds-ai model install --descriptor --directory Options: --directory Install into an application-managed directory @@ -62,6 +62,9 @@ function parseArguments(argv) { if (descriptor && model) throw new Error('Specify either a model name or --descriptor, not both'); if (!descriptor && !model) throw new Error('Specify a model name or --descriptor'); + if (descriptor && !directory) { + throw new Error('--descriptor requires --directory so the custom model can be configured'); + } return { descriptor, directory, model }; } diff --git a/lib/vector_embedding/embedding.js b/lib/vector_embedding/embedding.js index 003c86d..b1c09c8 100644 --- a/lib/vector_embedding/embedding.js +++ b/lib/vector_embedding/embedding.js @@ -1,17 +1,21 @@ import path from 'path'; +import { setTimeout as delay } from 'node:timers/promises'; import { Tensor } from './InferenceSession.js'; import { + MODEL_PROVISIONING_IN_PROGRESS, getModelCacheDir, getModelCacheRoot, loadModelAndTokenizer, modelDescriptorDigest, + provisionModel, readModelLock, - verifyModelDirectory, - validateModelDescriptor + verifyModelDirectory } from './model-utils.js'; import { DEFAULT_MODEL, findModelPreset, resolveModelPreset } from './models.js'; const STANDARD_INPUT_NAMES = new Set(['input_ids', 'attention_mask', 'token_type_ids']); +const MODEL_PROVISION_TIMEOUT_MS = 15 * 60 * 1000; +const MODEL_PROVISION_RETRY_MS = 250; async function createEmbeddingRuntime(configuration, options = {}) { const { model, modelDir } = await resolveEmbeddingModel(configuration, options); @@ -42,61 +46,132 @@ async function createEmbeddingRuntime(configuration, options = {}) { } async function resolveEmbeddingModel(configuration, options = {}) { - const { root = process.cwd(), env = process.env } = options; - if (configuration?.directory) { - const modelDir = path.resolve(root, configuration.directory); - if (typeof configuration.model !== 'string' || !configuration.model) { - throw new Error('embedding.model is required when embedding.directory is configured'); - } + const { + root = process.cwd(), + env = process.env, + fetchImpl, + resolvePreset = resolveModelPreset, + warn = (message) => console.warn(message) + } = options; + const { model: modelName, directory } = normalizeEmbeddingConfiguration(configuration); + + if (directory) { + const modelDir = path.resolve(root, directory); let model; try { model = await readModelLock(modelDir); } catch (error) { const recovery = /Embedding model lock not found/.test(error.message) - ? modelInstallHint(configuration.model, configuration.directory) + ? modelInstallHint(modelName, directory) : `Remove or replace the invalid lock explicitly, then ${lowercaseFirst( - modelInstallHint(configuration.model, configuration.directory) + modelInstallHint(modelName, directory) )}`; throw new Error(`${error.message}. ${recovery}`, { cause: error }); } - const preset = findModelPreset(configuration.model); + const preset = findModelPreset(modelName); if (preset && modelDescriptorDigest(model) !== modelDescriptorDigest(preset)) { throw new Error( - `Embedding model directory ${modelDir} does not contain the pinned ${configuration.model} preset. Choose another directory or remove it explicitly before provisioning it again.` + `Embedding model directory ${modelDir} does not contain the pinned ${modelName} preset. Choose another directory or remove it explicitly before provisioning it again.` ); } - if (!preset && model.repository !== configuration.model) { + if (!preset && model.repository !== modelName) { throw new Error( - `Embedding model directory ${modelDir} contains ${model.repository}, not ${configuration.model}. Choose another directory or remove it explicitly before provisioning it again.` + `Embedding model directory ${modelDir} contains ${model.repository}, not ${modelName}. Choose another directory or remove it explicitly before provisioning it again.` ); } try { await verifyModelDirectory(modelDir, model); } catch (error) { - throw new Error( - `${error.message}. ${modelInstallHint(configuration.model, configuration.directory)}`, - { - cause: error - } - ); + throw new Error(`${error.message}. ${modelInstallHint(modelName, directory)}`, { + cause: error + }); } return { model, modelDir }; } - const model = resolveConfiguredModel(configuration); + const model = resolvePreset(modelName); const modelDir = getModelCacheDir(getModelCacheRoot(env), model); try { await verifyModelDirectory(modelDir, model); } catch (error) { - throw new Error(`${error.message}. ${modelInstallHint(model.repository)}`, { - cause: error - }); + if (!/Embedding model is not provisioned or failed integrity checks/.test(error.message)) { + throw error; + } + warn( + `Embedding model '${model.repository}' is not available in '${modelDir}'. Downloading it now; application startup may be delayed. ${modelInstallHint(model.repository)}` + ); + try { + await provisionModelOnDemand(modelDir, model, { + env, + fetchImpl, + timeoutMs: options.provisionTimeoutMs, + retryMs: options.provisionRetryMs + }); + } catch (provisioningError) { + throw new Error( + `Failed to download embedding model '${model.repository}': ${provisioningError.message}. ${modelInstallHint(model.repository)}`, + { cause: provisioningError } + ); + } } return { model, modelDir }; } +async function provisionModelOnDemand(modelDir, model, options) { + const timeoutMs = options.timeoutMs ?? MODEL_PROVISION_TIMEOUT_MS; + const retryMs = options.retryMs ?? MODEL_PROVISION_RETRY_MS; + const deadline = Date.now() + timeoutMs; + + while (true) { + try { + // eslint-disable-next-line no-await-in-loop + return await provisionModel(model, { + directory: modelDir, + env: options.env, + fetchImpl: options.fetchImpl + }); + } catch (error) { + if (error.code !== MODEL_PROVISIONING_IN_PROGRESS) throw error; + const remaining = deadline - Date.now(); + if (remaining <= 0) { + throw new Error(`Timed out waiting for embedding model provisioning in ${modelDir}`, { + cause: error + }); + } + // eslint-disable-next-line no-await-in-loop + await delay(Math.min(retryMs, remaining)); + } + } +} + +function normalizeEmbeddingConfiguration(configuration) { + if (configuration == null) return { model: DEFAULT_MODEL.repository }; + if (typeof configuration !== 'object' || Array.isArray(configuration)) { + throw new TypeError('embedding must be an object with model and optional directory'); + } + + const unsupported = Object.keys(configuration).filter( + (name) => name !== 'model' && name !== 'directory' + ); + if (unsupported.length > 0) { + throw new Error( + `Unsupported embedding configuration: ${unsupported.join(', ')}. Only model and directory are supported.` + ); + } + if (typeof configuration.model !== 'string' || !configuration.model.trim()) { + throw new Error('embedding.model must be a non-empty string'); + } + if ( + configuration.directory !== undefined && + (typeof configuration.directory !== 'string' || !configuration.directory.trim()) + ) { + throw new Error('embedding.directory must be a non-empty string'); + } + return { model: configuration.model, directory: configuration.directory }; +} + function modelInstallHint(repository, directory) { const modelArgument = findModelPreset(repository) ? repository @@ -109,13 +184,6 @@ function lowercaseFirst(value) { return `${value[0].toLowerCase()}${value.slice(1)}`; } -function resolveConfiguredModel(configuration) { - if (configuration == null) return DEFAULT_MODEL; - if (typeof configuration === 'string') return resolveModelPreset(configuration); - if (configuration.model) return resolveModelPreset(configuration.model); - return validateModelDescriptor(configuration); -} - function createTokenizerState(tokenizer, maxLength) { const probeText = 'embedding tokenizer boundary probe'; const content = normalizeEncoding( diff --git a/lib/vector_embedding/model-utils.js b/lib/vector_embedding/model-utils.js index 5da2609..c3a2991 100644 --- a/lib/vector_embedding/model-utils.js +++ b/lib/vector_embedding/model-utils.js @@ -8,6 +8,7 @@ const DOWNLOAD_TIMEOUT_MS = 5 * 60 * 1000; const MODEL_LOCK_FILE = 'embedding.lock.json'; const MODEL_INSTALL_LOCK_FILE = '.embedding.install.lock'; const MODEL_LOCK_VERSION = 1; +const MODEL_PROVISIONING_IN_PROGRESS = 'ERR_EMBEDDING_MODEL_PROVISIONING_IN_PROGRESS'; const INSTALL_LOCK_STALE_MS = 30 * 60 * 1000; const PROVISIONED_DIRECTORY_MODE = 0o755; const PROVISIONED_FILE_MODE = 0o644; @@ -608,9 +609,12 @@ async function acquireInstallLock(lockPath, directory, owner) { if (await recoverStaleInstallLock(lockPath)) { return createInstallLock(lockPath, owner); } - throw new Error(`Embedding model directory ${directory} is already being provisioned`, { - cause: error - }); + throw Object.assign( + new Error(`Embedding model directory ${directory} is already being provisioned`, { + cause: error + }), + { code: MODEL_PROVISIONING_IN_PROGRESS } + ); } } @@ -756,6 +760,7 @@ async function loadModelAndTokenizer(modelDir, model) { export { MODEL_LOCK_FILE, + MODEL_PROVISIONING_IN_PROGRESS, downloadFile, downloadModelIfNeeded, artifactSetDigest, diff --git a/lib/vector_embedding/models.js b/lib/vector_embedding/models.js index c3c08b3..bfe6772 100644 --- a/lib/vector_embedding/models.js +++ b/lib/vector_embedding/models.js @@ -43,7 +43,7 @@ function resolveModelPreset(name) { const model = findModelPreset(name); if (!model) { throw new Error( - `Unsupported embedding model '${name}'. Use --descriptor for a compatible custom model.` + `Unsupported embedding model '${name}' for ad-hoc download. Provision a compatible custom model with --descriptor and --directory.` ); } return model; diff --git a/tests/model-provisioning.test.js b/tests/model-provisioning.test.js index 095858d..b47a5de 100644 --- a/tests/model-provisioning.test.js +++ b/tests/model-provisioning.test.js @@ -8,6 +8,7 @@ import { runModelCommand } from '../lib/vector_embedding/cli.js'; import { resolveEmbeddingModel } from '../lib/vector_embedding/embedding.js'; import { MODEL_LOCK_FILE, + getModelCacheDir, provisionModel, readModelLock, verifyModelDirectory @@ -32,6 +33,21 @@ describe('model presets', () => { test('rejects unknown model names', () => { assert.throws(() => resolveModelPreset('example/unknown'), /Unsupported embedding model/); }); + + test('accepts only model and directory in runtime configuration', async () => { + await assert.rejects( + resolveEmbeddingModel(DEFAULT_MODEL.repository), + /embedding must be an object with model and optional directory/ + ); + await assert.rejects( + resolveEmbeddingModel({ ...DEFAULT_MODEL }), + /Only model and directory are supported/ + ); + await assert.rejects( + resolveEmbeddingModel({ model: DEFAULT_MODEL.repository, directory: '' }), + /embedding.directory must be a non-empty string/ + ); + }); }); describe('explicit model provisioning', () => { @@ -294,30 +310,125 @@ describe('explicit model provisioning', () => { assert.deepEqual(await fs.readdir(directory), []); }); - test('keeps runtime initialization offline', async () => { + test('downloads a missing model into the default cache and reuses it', async () => { + const cacheRoot = await createTemporaryDirectory(); + const content = Buffer.from('lazy download fixture'); + const model = fixtureModel(content); + const requestedUrls = []; + const warnings = []; + const options = { + env: { CDS_AI_MODEL_CACHE: cacheRoot }, + fetchImpl: createFetch(content, requestedUrls), + resolvePreset(name) { + assert.equal(name, model.repository); + return model; + }, + warn: (message) => warnings.push(message) + }; + + const first = await resolveEmbeddingModel({ model: model.repository }, options); + const expectedDirectory = getModelCacheDir(cacheRoot, model); + + assert.equal(first.model, model); + assert.equal(first.modelDir, expectedDirectory); + assert.equal(warnings.length, 1); + assert.match(warnings[0], /Downloading it now; application startup may be delayed/); + assert.equal(requestedUrls.length, model.files.length); + assert.deepEqual(await readModelLock(expectedDirectory), model); + + const second = await resolveEmbeddingModel({ model: model.repository }, options); + assert.equal(second.modelDir, expectedDirectory); + assert.equal(warnings.length, 1); + assert.equal(requestedUrls.length, model.files.length); + }); + + test('waits for concurrent lazy provisioning and reuses the completed download', async () => { + const cacheRoot = await createTemporaryDirectory(); + const content = Buffer.from('concurrent lazy download fixture'); + const model = fixtureModel(content); + const requestedUrls = []; + const warnings = []; + let releaseDownload; + let signalDownloadStarted; + let firstRequest = true; + const downloadStarted = new Promise((resolve) => { + signalDownloadStarted = resolve; + }); + const waitForRelease = new Promise((resolve) => { + releaseDownload = resolve; + }); + const fetchImpl = async (url) => { + requestedUrls.push(url); + if (firstRequest) { + firstRequest = false; + signalDownloadStarted(); + await waitForRelease; + } + return new Response(content, { + headers: { 'content-length': String(content.length) } + }); + }; + const options = { + env: { CDS_AI_MODEL_CACHE: cacheRoot }, + fetchImpl, + resolvePreset: () => model, + warn: (message) => warnings.push(message), + provisionRetryMs: 5, + provisionTimeoutMs: 1000 + }; + + const first = resolveEmbeddingModel({ model: model.repository }, options); + await downloadStarted; + const second = resolveEmbeddingModel({ model: model.repository }, options); + await new Promise((resolve) => setTimeout(resolve, 20)); + releaseDownload(); + + const resolved = await Promise.all([first, second]); + assert.equal(resolved[0].modelDir, resolved[1].modelDir); + assert.equal(warnings.length, 2); + assert.equal(requestedUrls.length, model.files.length); + }); + + test('keeps explicitly configured directories offline', async () => { const root = await createTemporaryDirectory(); let fetched = false; - const originalFetch = globalThis.fetch; - globalThis.fetch = () => { - fetched = true; - throw new Error('runtime must not fetch'); - }; + await assert.rejects( + resolveEmbeddingModel( + { + model: DEFAULT_MODEL.repository, + directory: './models/minilm' + }, + { + root, + fetchImpl: () => { + fetched = true; + throw new Error('explicit directories must not fetch'); + } + } + ), + /cds-ai model install Xenova\/all-MiniLM-L6-v2 --directory \.\/models\/minilm/ + ); + assert.equal(fetched, false); + }); - try { - await assert.rejects( - resolveEmbeddingModel( - { - model: DEFAULT_MODEL.repository, - directory: './models/minilm' - }, - { root } - ), - /cds-ai model install Xenova\/all-MiniLM-L6-v2 --directory \.\/models\/minilm/ - ); - assert.equal(fetched, false); - } finally { - globalThis.fetch = originalFetch; - } + test('resolves relative directories from cds.root and preserves absolute directories', async () => { + const root = await createTemporaryDirectory(); + const directory = path.join(root, 'models', 'custom'); + const content = Buffer.from('directory resolution fixture'); + const model = fixtureModel(content); + await provisionModel(model, { directory, fetchImpl: createFetch(content) }); + + const relative = await resolveEmbeddingModel( + { model: model.repository, directory: './models/custom' }, + { root } + ); + const absolute = await resolveEmbeddingModel( + { model: model.repository, directory }, + { root: await createTemporaryDirectory() } + ); + + assert.equal(relative.modelDir, directory); + assert.equal(absolute.modelDir, directory); }); test('requires the complete built-in preset in an explicitly configured directory', async () => { @@ -375,6 +486,13 @@ describe('explicit model provisioning', () => { assert.deepEqual(await readModelLock(directory), model); assert.match(output.join(''), /Provisioned example\/model/); }); + + test('requires a directory when provisioning a custom descriptor', async () => { + await assert.rejects( + runModelCommand(['model', 'install', '--descriptor', './embedding-model.json']), + /--descriptor requires --directory/ + ); + }); }); function createFetch(content, requestedUrls = []) { diff --git a/tests/vector.test.js b/tests/vector.test.js index 1ac3ff8..eb59dab 100644 --- a/tests/vector.test.js +++ b/tests/vector.test.js @@ -3,7 +3,6 @@ import assert from 'node:assert'; import cds from '@sap/cds'; import { initializeEmbedding, vector_embedding } from '../lib/vector_embedding/index.js'; import { DEFAULT_MODEL } from '../lib/vector_embedding/embedding.js'; -import { getModelCacheDir, getModelCacheRoot } from '../lib/vector_embedding/model-utils.js'; let embeddingModule; @@ -134,8 +133,7 @@ describe('ai-sqlite integration', () => { db = await cds.connect.to('vector-db', { kind: 'ai-sqlite', embedding: { - model: DEFAULT_MODEL.repository, - directory: getModelCacheDir(getModelCacheRoot(), DEFAULT_MODEL) + model: DEFAULT_MODEL.repository }, credentials: { url: ':memory:' } }); @@ -162,43 +160,16 @@ describe('ai-sqlite integration', () => { assert.strictEqual(row.embedding, null); }); - test('validates a model descriptor supplied through the service options', async () => { + test('rejects model descriptors supplied through the service options', async () => { await assert.rejects( cds.connect.to('invalid-vector-db', { kind: 'ai-sqlite', embedding: { ...DEFAULT_MODEL, revision: 'main' }, credentials: { url: ':memory:' } }), - /immutable 40-64 character commit hash/ + /Only model and directory are supported/ ); }); - - test('keeps custom model behavior scoped to its service', async () => { - const customDb = await cds.connect.to('unnormalized-vector-db', { - kind: 'ai-sqlite', - embedding: { - ...DEFAULT_MODEL, - output: { ...DEFAULT_MODEL.output, normalize: false } - }, - credentials: { url: ':memory:' } - }); - - try { - const [defaultRow] = await db.run( - `SELECT VECTOR_EMBEDDING('Hello world', 'DOCUMENT', 'SAP_GXY.20250407') AS embedding` - ); - const [customRow] = await customDb.run( - `SELECT VECTOR_EMBEDDING('Hello world', 'DOCUMENT', 'SAP_GXY.20250407') AS embedding` - ); - const defaultNorm = vectorNorm(JSON.parse(defaultRow.embedding)); - const customNorm = vectorNorm(JSON.parse(customRow.embedding)); - - assert.ok(Math.abs(defaultNorm - 1) < 1e-5); - assert.ok(Math.abs(customNorm - 1) > 1e-3); - } finally { - await customDb.disconnect(); - } - }); }); // Helper function to calculate cosine similarity between two vectors @@ -217,7 +188,3 @@ function cosineSimilarity(a, b) { return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)); } - -function vectorNorm(vector) { - return Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0)); -} From 0677964019598751df7e551c45915a85dd712998 Mon Sep 17 00:00:00 2001 From: Sebastian Van Syckel Date: Thu, 27 Aug 2026 09:13:37 +0200 Subject: [PATCH 3/6] docs: explain embedding model provisioning --- README.md | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 69 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5ddecd2..bbb5ffa 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,9 @@ npm add @cap-js/sqlite onnxruntime-node@1.20.1 `ai-sqlite` currently requires exactly `onnxruntime-node` 1.20.1 because synchronous SQLite functions need a version-specific native runtime API. -Select `ai-sqlite` and the embedding model by name: +#### Model provisioning + +Runtime configuration is intentionally limited to a model name and an optional directory: ```json { @@ -236,15 +238,77 @@ Select `ai-sqlite` and the embedding model by name: } ``` -`Xenova/all-MiniLM-L6-v2` is used when the complete `embedding` option is omitted. If `embedding.directory` is not configured, startup checks the default model cache. A missing model is downloaded there after printing a warning, and subsequent starts reuse the verified files. +The complete `embedding` option can be omitted to use `Xenova/all-MiniLM-L6-v2`. When an `embedding` object is provided, `model` is required and `directory` is optional. No revision, dimensions, tokenizer, file, pooling, or checksum settings are accepted in runtime configuration. + +The provisioning approaches are: + +| Approach | `embedding` configuration | Startup behavior | +| --- | --- | --- | +| Ad-hoc download | `{ "model": "Xenova/all-MiniLM-L6-v2" }` or omit `embedding` for the default | Checks the automatically determined default cache directory. If the model is missing or invalid, logs a warning, provisions a verified copy, and reuses it on subsequent starts. | +| Pre-installed default cache | Same model-only configuration as ad-hoc download | Finds the model in the automatically determined default cache directory; no startup download is needed. | +| Pre-installed configured directory | `{ "model": "...", "directory": "..." }` | Reads and verifies the provisioned directory. Startup never downloads into or modifies it. | + +##### Ad-hoc download + +Ad-hoc download is available for built-in model presets. With no `directory`, the model name selects the preset and automatically determines: + +- the immutable model revision and verified artifact set +- embedding dimensions, tokenizer input limit, pooling, and normalization +- the default cache directory + +If the verified files are absent or fail their integrity checks, startup prints a warning and attempts to provision a verified copy. Concurrent processes using the same cold cache wait for the first download and then reuse it. A conflicting or malformed lock is never silently replaced. + +The cache root is selected from `CDS_AI_MODEL_CACHE` when set. Otherwise it uses `XDG_DATA_HOME/semantic-search/models` or `~/.local/share/semantic-search/models` on POSIX systems, and `LOCALAPPDATA/semantic-search/models`, `APPDATA/semantic-search/models`, or `~/AppData/Local/semantic-search/models` on Windows. The model-specific subdirectory is derived from its repository, revision, and artifact set. + +##### Pre-installed in the default cache -To avoid a download during application startup, provision the model explicitly first: +To avoid a runtime download while retaining model-only configuration, install the model before starting or deploying the application: ```sh npx cds-ai model install Xenova/all-MiniLM-L6-v2 ``` -The command uses the same default cache location as the runtime, verifies artifact sizes and SHA-256 checksums, and writes an `embedding.lock.json`. +The command uses the same automatically determined cache directory as the runtime, verifies artifact sizes and SHA-256 checksums, and writes an `embedding.lock.json`. The application configuration remains: + +```json +{ + "embedding": { + "model": "Xenova/all-MiniLM-L6-v2" + } +} +``` + +Use the same `CDS_AI_MODEL_CACHE` value during installation and at runtime when selecting a non-default cache root. + +##### Pre-installed in a configured directory + +An explicit directory is suitable for application images, read-only mounts, or a model shared by multiple applications: + +```sh +npx cds-ai model install Xenova/all-MiniLM-L6-v2 --directory ./models/minilm +``` + +```json +{ + "embedding": { + "model": "Xenova/all-MiniLM-L6-v2", + "directory": "./models/minilm" + } +} +``` + +The CLI resolves a relative `--directory` from its working directory. Runtime configuration resolves a relative `embedding.directory` from `cds.root`; absolute directories are used unchanged in both cases. Run the install command from `cds.root` or use the same absolute path so both refer to the same model directory. + +A configured directory must already contain a valid `embedding.lock.json` and all verified artifacts. Startup remains offline and fails rather than downloading if the directory is incomplete. `CDS_AI_MODEL_CACHE` has no effect when `embedding.directory` is configured. + +##### Automatic model metadata detection + +The runtime obtains technical model configuration without exposing it through `cds.requires.db.embedding`: + +- Without `directory`, the model name resolves to a built-in preset containing the pinned revision, artifacts, checksums, dimensions, tokenizer limit, and output semantics. +- With `directory`, the runtime reads those values from `embedding.lock.json`, verifies the artifacts, and checks that its repository matches `embedding.model`. Built-in presets are additionally checked against their complete pinned descriptor. + +This is not dynamic discovery of arbitrary Hugging Face repositories. Model-name-only ad-hoc downloads require a built-in preset. Other compatible models must be explicitly provisioned from a descriptor into a configured directory. The HANA-compatible SQL function can then be used in CQL: @@ -339,7 +403,7 @@ Then point the service at that locked directory. Runtime configuration contains } ``` -Relative directories are resolved from `cds.root`. Absolute directories are used unchanged, which allows multiple applications to reuse a shared model directory. When `embedding.directory` is configured, startup never downloads or modifies that directory: provision it while building the application image or mount an already provisioned directory. Provisioning canonicalizes symlinked parent directories and rejects a model directory that is itself a symlink. +Provisioning canonicalizes symlinked parent directories and rejects a model directory that is itself a symlink. The runtime accepts no model metadata beyond `embedding.model` and `embedding.directory`. Ad-hoc downloads by model name are available only for built-in presets. Custom models must first be installed from a descriptor into an explicitly configured directory. From 954d4f22e927b7808f27c35f57d5dea5a5c172df Mon Sep 17 00:00:00 2001 From: Sebastian Van Syckel Date: Thu, 27 Aug 2026 10:41:18 +0200 Subject: [PATCH 4/6] fix: require explicit embedding model --- CHANGELOG.md | 2 +- README.md | 22 +++++++++++----------- lib/vector_embedding/embedding.js | 27 ++++++++++++++------------- lib/vector_embedding/index.js | 21 +++++++++++---------- lib/vector_embedding/models.js | 6 +++--- tests/model-provisioning.test.js | 30 +++++++++++++++++++----------- tests/vector.test.js | 24 +++++++++++++++++++----- 7 files changed, 78 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cebda4c..93254bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ ### Added - Add the `ai-sqlite` kind with a `VECTOR_EMBEDDING` function using ONNX Runtime and the `Xenova/all-MiniLM-L6-v2` model (384 dimensions) - - Adds `cds-ai model install` for explicit, checksum-verified model provisioning and a warned, on-demand download into the default cache + - Requires `cds.env.requires.db.embedding.model`; adds `cds-ai model install` for explicit, checksum-verified model provisioning and a warned, on-demand download of an explicitly named built-in model into the automatic cache - Uses `@huggingface/tokenizers` and chunks long input without dropping per-chunk special tokens - Configures embedding runtimes only through `model` and an optional relative or absolute `directory`; custom model metadata remains in the provisioned lock - Supports both 3-parameter `(text, text_type, model_and_version)` and 4-parameter variants with `remote_source` diff --git a/README.md b/README.md index bbb5ffa..5ea1373 100644 --- a/README.md +++ b/README.md @@ -207,7 +207,7 @@ resources: ### 3. Local Vector Embeddings with SQLite -The `ai-sqlite` database kind extends `@cap-js/sqlite` with local semantic embeddings using an ONNX encoder model. It uses the pinned `Xenova/all-MiniLM-L6-v2` model by default. +The `ai-sqlite` database kind extends `@cap-js/sqlite` with local semantic embeddings using an ONNX encoder model. Configure the model explicitly for every service. #### Usage @@ -238,14 +238,14 @@ Runtime configuration is intentionally limited to a model name and an optional d } ``` -The complete `embedding` option can be omitted to use `Xenova/all-MiniLM-L6-v2`. When an `embedding` object is provided, `model` is required and `directory` is optional. No revision, dimensions, tokenizer, file, pooling, or checksum settings are accepted in runtime configuration. +`embedding` and its `model` are required. If `cds.env.requires.db.embedding.model` is absent, `ai-sqlite` fails during startup. `directory` is optional. No revision, dimensions, tokenizer, file, pooling, or checksum settings are accepted in runtime configuration. The provisioning approaches are: | Approach | `embedding` configuration | Startup behavior | | --- | --- | --- | -| Ad-hoc download | `{ "model": "Xenova/all-MiniLM-L6-v2" }` or omit `embedding` for the default | Checks the automatically determined default cache directory. If the model is missing or invalid, logs a warning, provisions a verified copy, and reuses it on subsequent starts. | -| Pre-installed default cache | Same model-only configuration as ad-hoc download | Finds the model in the automatically determined default cache directory; no startup download is needed. | +| Ad-hoc download | `{ "model": "Xenova/all-MiniLM-L6-v2" }` | Checks the automatically determined cache directory. If the model is missing or invalid, logs a warning, provisions a verified copy, and reuses it on subsequent starts. | +| Pre-installed automatic cache | Same model-only configuration as ad-hoc download | Finds the model in the automatically determined cache directory; no startup download is needed. | | Pre-installed configured directory | `{ "model": "...", "directory": "..." }` | Reads and verifies the provisioned directory. Startup never downloads into or modifies it. | ##### Ad-hoc download @@ -254,13 +254,13 @@ Ad-hoc download is available for built-in model presets. With no `directory`, th - the immutable model revision and verified artifact set - embedding dimensions, tokenizer input limit, pooling, and normalization -- the default cache directory +- the automatically determined cache directory If the verified files are absent or fail their integrity checks, startup prints a warning and attempts to provision a verified copy. Concurrent processes using the same cold cache wait for the first download and then reuse it. A conflicting or malformed lock is never silently replaced. The cache root is selected from `CDS_AI_MODEL_CACHE` when set. Otherwise it uses `XDG_DATA_HOME/semantic-search/models` or `~/.local/share/semantic-search/models` on POSIX systems, and `LOCALAPPDATA/semantic-search/models`, `APPDATA/semantic-search/models`, or `~/AppData/Local/semantic-search/models` on Windows. The model-specific subdirectory is derived from its repository, revision, and artifact set. -##### Pre-installed in the default cache +##### Pre-installed in the automatic cache To avoid a runtime download while retaining model-only configuration, install the model before starting or deploying the application: @@ -278,7 +278,7 @@ The command uses the same automatically determined cache directory as the runtim } ``` -Use the same `CDS_AI_MODEL_CACHE` value during installation and at runtime when selecting a non-default cache root. +Use the same `CDS_AI_MODEL_CACHE` value during installation and at runtime when selecting another cache root. ##### Pre-installed in a configured directory @@ -326,13 +326,13 @@ SELECT.from('Books').columns` **Returns:** -- JSON stringified array of embedding values (384 dimensions for the default MiniLM model; custom models use their configured `dimensions`) +- JSON stringified array of embedding values with the configured model's dimensions **Features:** - **Initialization**: The ONNX model is loaded when the `ai-sqlite` service starts - **Flexible provisioning**: Preinstall models with `cds-ai model install`, or let development startup download a missing built-in model after warning you -- **Verified cache**: The pinned model revision and artifact set are stored by default below the user's data directory; set `CDS_AI_MODEL_CACHE` to select another cache root +- **Verified cache**: The selected preset's pinned revision and artifact set are stored below the user's data directory unless `CDS_AI_MODEL_CACHE` selects another cache root - **Hugging Face tokenization**: Uses `@huggingface/tokenizers` and safely chunks text that exceeds the model limit - **Deterministic**: Same input always produces same output - **Normalized vectors**: MiniLM embeddings are L2-normalized; custom descriptors control this with `output.normalize` @@ -411,8 +411,8 @@ Compatible models must accept `input_ids` and may additionally accept `attention **Error Handling:** -- Starting `ai-sqlite` fails if the ONNX model cannot be initialized -- A missing built-in model in the default cache is downloaded after a startup warning +- Starting `ai-sqlite` fails if `cds.env.requires.db.embedding.model` is not set or the ONNX model cannot be initialized +- A missing explicitly named built-in model in the automatic cache is downloaded after a startup warning - Starting `ai-sqlite` fails with a provisioning command if a configured model directory is missing or fails integrity checks - Provisioning downloads are time-limited and accepted only when their expected size and SHA-256 match - Throws if embedding generation fails diff --git a/lib/vector_embedding/embedding.js b/lib/vector_embedding/embedding.js index b1c09c8..145924a 100644 --- a/lib/vector_embedding/embedding.js +++ b/lib/vector_embedding/embedding.js @@ -11,7 +11,7 @@ import { readModelLock, verifyModelDirectory } from './model-utils.js'; -import { DEFAULT_MODEL, findModelPreset, resolveModelPreset } from './models.js'; +import { findModelPreset, resolveModelPreset } from './models.js'; const STANDARD_INPUT_NAMES = new Set(['input_ids', 'attention_mask', 'token_type_ids']); const MODEL_PROVISION_TIMEOUT_MS = 15 * 60 * 1000; @@ -147,7 +147,9 @@ async function provisionModelOnDemand(modelDir, model, options) { } function normalizeEmbeddingConfiguration(configuration) { - if (configuration == null) return { model: DEFAULT_MODEL.repository }; + if (configuration == null) { + throw new Error('cds.env.requires.db.embedding.model must be a non-empty string'); + } if (typeof configuration !== 'object' || Array.isArray(configuration)) { throw new TypeError('embedding must be an object with model and optional directory'); } @@ -161,7 +163,7 @@ function normalizeEmbeddingConfiguration(configuration) { ); } if (typeof configuration.model !== 'string' || !configuration.model.trim()) { - throw new Error('embedding.model must be a non-empty string'); + throw new Error('cds.env.requires.db.embedding.model must be a non-empty string'); } if ( configuration.directory !== undefined && @@ -396,28 +398,28 @@ function validateAttentionMask(mask) { return mask; } -let defaultRuntime; -let defaultInitialization; +let sessionRuntime; +let sessionInitialization; -async function createSession() { - defaultInitialization ??= createEmbeddingRuntime() - .then((runtime) => (defaultRuntime = runtime)) +async function createSession(configuration, options) { + sessionInitialization ??= createEmbeddingRuntime(configuration, options) + .then((runtime) => (sessionRuntime = runtime)) .catch((error) => { - defaultInitialization = undefined; + sessionInitialization = undefined; throw error; }); - return defaultInitialization; + return sessionInitialization; } function embedding(text) { - if (!defaultRuntime) { + if (!sessionRuntime) { throw new Error( 'Embedding session not initialized. Call createSession() before using embedding().' ); } const chunk = { content: text }; return Object.defineProperty(chunk, 'embedding', { - value: defaultRuntime.embedding(text), + value: sessionRuntime.embedding(text), writable: true, configurable: true, enumerable: false @@ -425,7 +427,6 @@ function embedding(text) { } export { - DEFAULT_MODEL, createSession, createEmbeddingRuntime, createFeeds, diff --git a/lib/vector_embedding/index.js b/lib/vector_embedding/index.js index 4089a45..aad5cca 100644 --- a/lib/vector_embedding/index.js +++ b/lib/vector_embedding/index.js @@ -3,14 +3,11 @@ import * as embeddingModule from './embedding.js'; const LOG = cds.log('@cap-js/ai'); let loggedInitialization = false; -const { dimensions: DEFAULT_DIMENSIONS } = embeddingModule.DEFAULT_MODEL; -const modelDimensions = { - 'SAP_GXY.20250407': DEFAULT_DIMENSIONS, - 'SAP_GXY.20240715': DEFAULT_DIMENSIONS -}; +let dimensions; -async function initializeEmbedding() { - await embeddingModule.createSession(); +async function initializeEmbedding(configuration, options) { + const runtime = await embeddingModule.createSession(configuration, options); + dimensions = runtime.dimensions; if (!loggedInitialization) { LOG.info('Vector embedding ONNX model initialized'); loggedInitialization = true; @@ -20,10 +17,14 @@ async function initializeEmbedding() { function vector_embedding(text, text_type, model_and_version) { void text_type; // Retained for HANA-compatible function arity. + void model_and_version; // The configured embedding model determines vector dimensions. if (text) return JSON.stringify(Array.from(embeddingModule.embedding(text).embedding)); - return JSON.stringify( - new Array(modelDimensions[model_and_version] ?? DEFAULT_DIMENSIONS).fill(0) - ); + if (!dimensions) { + throw new Error( + 'Embedding session not initialized. Call initializeEmbedding() with an embedding configuration before using vector_embedding().' + ); + } + return JSON.stringify(new Array(dimensions).fill(0)); } export { initializeEmbedding, vector_embedding }; diff --git a/lib/vector_embedding/models.js b/lib/vector_embedding/models.js index bfe6772..a33d47e 100644 --- a/lib/vector_embedding/models.js +++ b/lib/vector_embedding/models.js @@ -1,4 +1,4 @@ -const DEFAULT_MODEL = Object.freeze({ +const MINILM_MODEL = Object.freeze({ repository: 'Xenova/all-MiniLM-L6-v2', revision: '751bff37182d3f1213fa05d7196b954e230abad9', dimensions: 384, @@ -33,7 +33,7 @@ const DEFAULT_MODEL = Object.freeze({ }) }); -const MODEL_PRESETS = new Map([[DEFAULT_MODEL.repository, DEFAULT_MODEL]]); +const MODEL_PRESETS = new Map([[MINILM_MODEL.repository, MINILM_MODEL]]); function findModelPreset(name) { return MODEL_PRESETS.get(name); @@ -49,4 +49,4 @@ function resolveModelPreset(name) { return model; } -export { DEFAULT_MODEL, findModelPreset, resolveModelPreset }; +export { MINILM_MODEL, findModelPreset, resolveModelPreset }; diff --git a/tests/model-provisioning.test.js b/tests/model-provisioning.test.js index b47a5de..af74279 100644 --- a/tests/model-provisioning.test.js +++ b/tests/model-provisioning.test.js @@ -13,7 +13,7 @@ import { readModelLock, verifyModelDirectory } from '../lib/vector_embedding/model-utils.js'; -import { DEFAULT_MODEL, resolveModelPreset } from '../lib/vector_embedding/models.js'; +import { MINILM_MODEL, resolveModelPreset } from '../lib/vector_embedding/models.js'; const temporaryDirectories = []; @@ -26,8 +26,8 @@ afterEach(async () => { }); describe('model presets', () => { - test('resolves the default MiniLM model by repository name', () => { - assert.equal(resolveModelPreset('Xenova/all-MiniLM-L6-v2'), DEFAULT_MODEL); + test('resolves the MiniLM model preset by repository name', () => { + assert.equal(resolveModelPreset('Xenova/all-MiniLM-L6-v2'), MINILM_MODEL); }); test('rejects unknown model names', () => { @@ -36,15 +36,23 @@ describe('model presets', () => { test('accepts only model and directory in runtime configuration', async () => { await assert.rejects( - resolveEmbeddingModel(DEFAULT_MODEL.repository), + resolveEmbeddingModel(), + /cds\.env\.requires\.db\.embedding\.model must be a non-empty string/ + ); + await assert.rejects( + resolveEmbeddingModel({}), + /cds\.env\.requires\.db\.embedding\.model must be a non-empty string/ + ); + await assert.rejects( + resolveEmbeddingModel(MINILM_MODEL.repository), /embedding must be an object with model and optional directory/ ); await assert.rejects( - resolveEmbeddingModel({ ...DEFAULT_MODEL }), + resolveEmbeddingModel({ ...MINILM_MODEL }), /Only model and directory are supported/ ); await assert.rejects( - resolveEmbeddingModel({ model: DEFAULT_MODEL.repository, directory: '' }), + resolveEmbeddingModel({ model: MINILM_MODEL.repository, directory: '' }), /embedding.directory must be a non-empty string/ ); }); @@ -310,7 +318,7 @@ describe('explicit model provisioning', () => { assert.deepEqual(await fs.readdir(directory), []); }); - test('downloads a missing model into the default cache and reuses it', async () => { + test('downloads a missing model into the automatic cache and reuses it', async () => { const cacheRoot = await createTemporaryDirectory(); const content = Buffer.from('lazy download fixture'); const model = fixtureModel(content); @@ -395,7 +403,7 @@ describe('explicit model provisioning', () => { await assert.rejects( resolveEmbeddingModel( { - model: DEFAULT_MODEL.repository, + model: MINILM_MODEL.repository, directory: './models/minilm' }, { @@ -436,12 +444,12 @@ describe('explicit model provisioning', () => { const content = Buffer.from('imposter preset fixture'); const model = { ...fixtureModel(content), - repository: DEFAULT_MODEL.repository + repository: MINILM_MODEL.repository }; await provisionModel(model, { directory, fetchImpl: createFetch(content) }); await assert.rejects( - resolveEmbeddingModel({ model: DEFAULT_MODEL.repository, directory }), + resolveEmbeddingModel({ model: MINILM_MODEL.repository, directory }), /does not contain the pinned Xenova\/all-MiniLM-L6-v2 preset/ ); }); @@ -460,7 +468,7 @@ describe('explicit model provisioning', () => { await fs.writeFile(path.join(directory, MODEL_LOCK_FILE), '{}'); await assert.rejects( - resolveEmbeddingModel({ model: DEFAULT_MODEL.repository, directory }), + resolveEmbeddingModel({ model: MINILM_MODEL.repository, directory }), /Remove or replace the invalid lock explicitly, then run 'npx cds-ai model install/ ); }); diff --git a/tests/vector.test.js b/tests/vector.test.js index eb59dab..3f5223d 100644 --- a/tests/vector.test.js +++ b/tests/vector.test.js @@ -2,12 +2,12 @@ import { after, before, describe, test } from 'node:test'; import assert from 'node:assert'; import cds from '@sap/cds'; import { initializeEmbedding, vector_embedding } from '../lib/vector_embedding/index.js'; -import { DEFAULT_MODEL } from '../lib/vector_embedding/embedding.js'; +import { MINILM_MODEL } from '../lib/vector_embedding/models.js'; let embeddingModule; before(async () => { - embeddingModule = await initializeEmbedding(); + embeddingModule = await initializeEmbedding({ model: MINILM_MODEL.repository }); }); describe('Vector embedding function (standalone)', () => { @@ -113,7 +113,11 @@ describe('Vector embedding function (standalone)', () => { const result3 = vector_embedding('test', 'DOCUMENT', 'unknown_model'); const embedding3 = JSON.parse(result3); - assert.strictEqual(embedding3.length, 384, 'Unknown model should default to 384 dimensions'); + assert.strictEqual( + embedding3.length, + 384, + 'Compatibility identifiers should use the configured model dimensions' + ); }); test('retains the embedding module wrapper', () => { @@ -129,11 +133,21 @@ describe('Vector embedding function (standalone)', () => { describe('ai-sqlite integration', () => { let db; + test('requires an explicitly configured embedding model during startup', async () => { + await assert.rejects( + cds.connect.to('missing-embedding-model-db', { + kind: 'ai-sqlite', + credentials: { url: ':memory:' } + }), + /cds\.env\.requires\.db\.embedding\.model must be a non-empty string/ + ); + }); + before(async () => { db = await cds.connect.to('vector-db', { kind: 'ai-sqlite', embedding: { - model: DEFAULT_MODEL.repository + model: MINILM_MODEL.repository }, credentials: { url: ':memory:' } }); @@ -164,7 +178,7 @@ describe('ai-sqlite integration', () => { await assert.rejects( cds.connect.to('invalid-vector-db', { kind: 'ai-sqlite', - embedding: { ...DEFAULT_MODEL, revision: 'main' }, + embedding: { ...MINILM_MODEL, revision: 'main' }, credentials: { url: ':memory:' } }), /Only model and directory are supported/ From de5622c64dbe53c0e08c6578161349a91ca89088 Mon Sep 17 00:00:00 2001 From: Sebastian Van Syckel Date: Thu, 27 Aug 2026 11:04:21 +0200 Subject: [PATCH 5/6] refactor: require provisioned embedding models --- .gitignore | 3 +- CHANGELOG.md | 6 +- README.md | 88 +++--------- lib/sqlite/AISQLiteService.js | 3 +- lib/vector_embedding/cli.js | 36 ++--- lib/vector_embedding/embedding.js | 141 ++++---------------- lib/vector_embedding/model-utils.js | 56 +------- lib/vector_embedding/models.js | 52 -------- package.json | 6 +- tests/fixtures/minilm.embedding-model.json | 34 +++++ tests/model-provisioning.test.js | 147 ++++++--------------- tests/vector-unit.test.js | 27 ---- tests/vector.test.js | 27 +++- 13 files changed, 163 insertions(+), 463 deletions(-) delete mode 100644 lib/vector_embedding/models.js create mode 100644 tests/fixtures/minilm.embedding-model.json diff --git a/.gitignore b/.gitignore index 4104bfe..2a91514 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ gen/ package-lock.json .env .cdsrc-private.json -resources/ \ No newline at end of file +resources/ +tests/.models/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 93254bc..f93e39c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,10 @@ ### Added -- Add the `ai-sqlite` kind with a `VECTOR_EMBEDDING` function using ONNX Runtime and the `Xenova/all-MiniLM-L6-v2` model (384 dimensions) - - Requires `cds.env.requires.db.embedding.model`; adds `cds-ai model install` for explicit, checksum-verified model provisioning and a warned, on-demand download of an explicitly named built-in model into the automatic cache +- Add the `ai-sqlite` kind with a `VECTOR_EMBEDDING` function using compatible ONNX encoder models + - Requires `cds.env.requires.db.embedding.model` and `directory`; adds `cds-ai model install` for explicit, checksum-verified model provisioning from a descriptor - Uses `@huggingface/tokenizers` and chunks long input without dropping per-chunk special tokens - - Configures embedding runtimes only through `model` and an optional relative or absolute `directory`; custom model metadata remains in the provisioned lock + - Configures embedding runtimes only through `model` and a relative or absolute `directory`; model metadata remains in the provisioned lock - Supports both 3-parameter `(text, text_type, model_and_version)` and 4-parameter variants with `remote_source` - Compatible with `SAP_GXY.20250407` and `SAP_GXY.20240715` model versions - Synchronous execution suitable for SQLite user-defined functions diff --git a/README.md b/README.md index 5ea1373..180d257 100644 --- a/README.md +++ b/README.md @@ -221,7 +221,7 @@ npm add @cap-js/sqlite onnxruntime-node@1.20.1 #### Model provisioning -Runtime configuration is intentionally limited to a model name and an optional directory: +Runtime configuration is intentionally limited to a model name and directory: ```json { @@ -230,7 +230,8 @@ Runtime configuration is intentionally limited to a model name and an optional d "db": { "kind": "ai-sqlite", "embedding": { - "model": "Xenova/all-MiniLM-L6-v2" + "model": "organization/model", + "directory": "./models/embedding" } } } @@ -238,77 +239,25 @@ Runtime configuration is intentionally limited to a model name and an optional d } ``` -`embedding` and its `model` are required. If `cds.env.requires.db.embedding.model` is absent, `ai-sqlite` fails during startup. `directory` is optional. No revision, dimensions, tokenizer, file, pooling, or checksum settings are accepted in runtime configuration. - -The provisioning approaches are: - -| Approach | `embedding` configuration | Startup behavior | -| --- | --- | --- | -| Ad-hoc download | `{ "model": "Xenova/all-MiniLM-L6-v2" }` | Checks the automatically determined cache directory. If the model is missing or invalid, logs a warning, provisions a verified copy, and reuses it on subsequent starts. | -| Pre-installed automatic cache | Same model-only configuration as ad-hoc download | Finds the model in the automatically determined cache directory; no startup download is needed. | -| Pre-installed configured directory | `{ "model": "...", "directory": "..." }` | Reads and verifies the provisioned directory. Startup never downloads into or modifies it. | - -##### Ad-hoc download - -Ad-hoc download is available for built-in model presets. With no `directory`, the model name selects the preset and automatically determines: - -- the immutable model revision and verified artifact set -- embedding dimensions, tokenizer input limit, pooling, and normalization -- the automatically determined cache directory - -If the verified files are absent or fail their integrity checks, startup prints a warning and attempts to provision a verified copy. Concurrent processes using the same cold cache wait for the first download and then reuse it. A conflicting or malformed lock is never silently replaced. - -The cache root is selected from `CDS_AI_MODEL_CACHE` when set. Otherwise it uses `XDG_DATA_HOME/semantic-search/models` or `~/.local/share/semantic-search/models` on POSIX systems, and `LOCALAPPDATA/semantic-search/models`, `APPDATA/semantic-search/models`, or `~/AppData/Local/semantic-search/models` on Windows. The model-specific subdirectory is derived from its repository, revision, and artifact set. - -##### Pre-installed in the automatic cache - -To avoid a runtime download while retaining model-only configuration, install the model before starting or deploying the application: - -```sh -npx cds-ai model install Xenova/all-MiniLM-L6-v2 -``` - -The command uses the same automatically determined cache directory as the runtime, verifies artifact sizes and SHA-256 checksums, and writes an `embedding.lock.json`. The application configuration remains: - -```json -{ - "embedding": { - "model": "Xenova/all-MiniLM-L6-v2" - } -} -``` - -Use the same `CDS_AI_MODEL_CACHE` value during installation and at runtime when selecting another cache root. +Both `embedding.model` and `embedding.directory` are required. `ai-sqlite` fails during startup when either is absent. No revision, dimensions, tokenizer, file, pooling, or checksum settings are accepted in runtime configuration. -##### Pre-installed in a configured directory - -An explicit directory is suitable for application images, read-only mounts, or a model shared by multiple applications: +Create an `embedding-model.json` descriptor for the model, then provision it before application startup: ```sh -npx cds-ai model install Xenova/all-MiniLM-L6-v2 --directory ./models/minilm +npx cds-ai model install --descriptor ./embedding-model.json --directory ./models/embedding ``` -```json -{ - "embedding": { - "model": "Xenova/all-MiniLM-L6-v2", - "directory": "./models/minilm" - } -} -``` +Provisioning downloads the descriptor's pinned artifacts, verifies their sizes and SHA-256 checksums, and writes the validated descriptor to `embedding.lock.json`. The lock content is therefore generated from the descriptor; it is not inferred from the model name. The CLI resolves a relative `--directory` from its working directory. Runtime configuration resolves a relative `embedding.directory` from `cds.root`; absolute directories are used unchanged in both cases. Run the install command from `cds.root` or use the same absolute path so both refer to the same model directory. -A configured directory must already contain a valid `embedding.lock.json` and all verified artifacts. Startup remains offline and fails rather than downloading if the directory is incomplete. `CDS_AI_MODEL_CACHE` has no effect when `embedding.directory` is configured. - -##### Automatic model metadata detection +A configured directory must already contain a valid `embedding.lock.json` and all verified artifacts. Startup remains offline and fails rather than downloading or modifying the directory if it is incomplete. -The runtime obtains technical model configuration without exposing it through `cds.requires.db.embedding`: +##### Model metadata -- Without `directory`, the model name resolves to a built-in preset containing the pinned revision, artifacts, checksums, dimensions, tokenizer limit, and output semantics. -- With `directory`, the runtime reads those values from `embedding.lock.json`, verifies the artifacts, and checks that its repository matches `embedding.model`. Built-in presets are additionally checked against their complete pinned descriptor. +At startup, the runtime reads the immutable revision, artifacts, checksums, dimensions, tokenizer limit, pooling, and normalization from `embedding.lock.json`. It verifies the files and checks that the lock's repository matches `embedding.model`. -This is not dynamic discovery of arbitrary Hugging Face repositories. Model-name-only ad-hoc downloads require a built-in preset. Other compatible models must be explicitly provisioned from a descriptor into a configured directory. +This is local metadata detection, not discovery from a model name or arbitrary Hugging Face repository. A model name alone cannot reliably determine artifact selection, pooling, normalization, or other runtime semantics. The HANA-compatible SQL function can then be used in CQL: @@ -331,16 +280,16 @@ SELECT.from('Books').columns` **Features:** - **Initialization**: The ONNX model is loaded when the `ai-sqlite` service starts -- **Flexible provisioning**: Preinstall models with `cds-ai model install`, or let development startup download a missing built-in model after warning you -- **Verified cache**: The selected preset's pinned revision and artifact set are stored below the user's data directory unless `CDS_AI_MODEL_CACHE` selects another cache root +- **Explicit provisioning**: Install models before startup with `cds-ai model install` +- **Verified artifacts**: The provisioned lock pins the revision, artifact sizes, and SHA-256 checksums - **Hugging Face tokenization**: Uses `@huggingface/tokenizers` and safely chunks text that exceeds the model limit - **Deterministic**: Same input always produces same output -- **Normalized vectors**: MiniLM embeddings are L2-normalized; custom descriptors control this with `output.normalize` +- **Configurable output handling**: The descriptor controls pooling and L2 normalization - **Semantic similarity**: Embeddings capture text meaning for similarity search -#### Compatible custom encoder models +#### Compatible encoder models -The built-in preset currently recognizes `Xenova/all-MiniLM-L6-v2`. For a compatible custom model, create an `embedding-model.json` descriptor. Models are not discovered dynamically: every artifact must belong to an immutable revision and have an expected size and SHA-256 checksum. +Create an `embedding-model.json` descriptor for each compatible model. Models are not discovered dynamically: every artifact must belong to an immutable revision and have an expected size and SHA-256 checksum. ```json { @@ -405,14 +354,13 @@ Then point the service at that locked directory. Runtime configuration contains Provisioning canonicalizes symlinked parent directories and rejects a model directory that is itself a symlink. -The runtime accepts no model metadata beyond `embedding.model` and `embedding.directory`. Ad-hoc downloads by model name are available only for built-in presets. Custom models must first be installed from a descriptor into an explicitly configured directory. +The runtime accepts no model metadata beyond `embedding.model` and `embedding.directory`. Every model must first be installed from a descriptor into the configured directory. Compatible models must accept `input_ids` and may additionally accept `attention_mask` and `token_type_ids`, all as `int64` tensors. Their configured float32 or float64 output must support `mean` or `cls` pooling from `[1, sequence, dimensions]`, or `none` for an already pooled `[dimensions]` or `[1, dimensions]` tensor. Additional pinned ONNX data files can use the `auxiliary` role. Startup probes the model and rejects incompatible input names, output names, types, shapes, or dimensions. **Error Handling:** -- Starting `ai-sqlite` fails if `cds.env.requires.db.embedding.model` is not set or the ONNX model cannot be initialized -- A missing explicitly named built-in model in the automatic cache is downloaded after a startup warning +- Starting `ai-sqlite` fails if `cds.env.requires.db.embedding.model` or `cds.env.requires.db.embedding.directory` is not set, or the ONNX model cannot be initialized - Starting `ai-sqlite` fails with a provisioning command if a configured model directory is missing or fails integrity checks - Provisioning downloads are time-limited and accepted only when their expected size and SHA-256 match - Throws if embedding generation fails diff --git a/lib/sqlite/AISQLiteService.js b/lib/sqlite/AISQLiteService.js index 7473f47..07212ac 100644 --- a/lib/sqlite/AISQLiteService.js +++ b/lib/sqlite/AISQLiteService.js @@ -7,8 +7,7 @@ const LOG = cds.log('@cap-js/ai'); export default class AISQLiteService extends SQLiteService { async init() { this._embeddingRuntime = await createEmbeddingRuntime(this.options.embedding, { - root: cds.root, - warn: (message) => LOG.warn(message) + root: cds.root }); LOG.info('Vector embedding ONNX model initialized'); return super.init(); diff --git a/lib/vector_embedding/cli.js b/lib/vector_embedding/cli.js index 07f2543..0b162a2 100644 --- a/lib/vector_embedding/cli.js +++ b/lib/vector_embedding/cli.js @@ -1,39 +1,28 @@ import fs from 'node:fs/promises'; import path from 'node:path'; -import { - getModelCacheDir, - getModelCacheRoot, - provisionModel, - validateModelDescriptor -} from './model-utils.js'; -import { resolveModelPreset } from './models.js'; +import { provisionModel, validateModelDescriptor } from './model-utils.js'; const HELP = `Usage: - cds-ai model install [--directory ] cds-ai model install --descriptor --directory Options: --directory Install into an application-managed directory - --descriptor Read a compatible custom model descriptor from JSON + --descriptor Read a compatible model descriptor from JSON --help Show this help `; async function runModelCommand(argv, options = {}) { - const { cwd = process.cwd(), env = process.env, fetchImpl, stdout = process.stdout } = options; + const { cwd = process.cwd(), fetchImpl, stdout = process.stdout } = options; const command = parseArguments(argv); if (command.help) { stdout.write(HELP); return; } - const model = command.descriptor - ? await readDescriptor(path.resolve(cwd, command.descriptor)) - : resolveModelPreset(command.model); - const directory = command.directory - ? path.resolve(cwd, command.directory) - : getModelCacheDir(getModelCacheRoot(env), model); + const model = await readDescriptor(path.resolve(cwd, command.descriptor)); + const directory = path.resolve(cwd, command.directory); - await provisionModel(model, { directory, env, fetchImpl }); + await provisionModel(model, { directory, fetchImpl }); stdout.write(`Provisioned ${model.repository} in ${directory}\n`); } @@ -43,7 +32,6 @@ function parseArguments(argv) { throw new Error(`Unsupported command.\n\n${HELP}`); } - let model; let descriptor; let directory; for (let index = 2; index < argv.length; index++) { @@ -56,16 +44,12 @@ function parseArguments(argv) { continue; } if (argument.startsWith('-')) throw new Error(`Unknown option '${argument}'`); - if (model) throw new Error(`Unexpected argument '${argument}'`); - model = argument; + throw new Error(`Unexpected argument '${argument}'`); } - if (descriptor && model) throw new Error('Specify either a model name or --descriptor, not both'); - if (!descriptor && !model) throw new Error('Specify a model name or --descriptor'); - if (descriptor && !directory) { - throw new Error('--descriptor requires --directory so the custom model can be configured'); - } - return { descriptor, directory, model }; + if (!descriptor) throw new Error('--descriptor is required'); + if (!directory) throw new Error('--directory is required'); + return { descriptor, directory }; } async function readDescriptor(file) { diff --git a/lib/vector_embedding/embedding.js b/lib/vector_embedding/embedding.js index 145924a..c42a153 100644 --- a/lib/vector_embedding/embedding.js +++ b/lib/vector_embedding/embedding.js @@ -1,21 +1,8 @@ import path from 'path'; -import { setTimeout as delay } from 'node:timers/promises'; import { Tensor } from './InferenceSession.js'; -import { - MODEL_PROVISIONING_IN_PROGRESS, - getModelCacheDir, - getModelCacheRoot, - loadModelAndTokenizer, - modelDescriptorDigest, - provisionModel, - readModelLock, - verifyModelDirectory -} from './model-utils.js'; -import { findModelPreset, resolveModelPreset } from './models.js'; +import { loadModelAndTokenizer, readModelLock, verifyModelDirectory } from './model-utils.js'; const STANDARD_INPUT_NAMES = new Set(['input_ids', 'attention_mask', 'token_type_ids']); -const MODEL_PROVISION_TIMEOUT_MS = 15 * 60 * 1000; -const MODEL_PROVISION_RETRY_MS = 250; async function createEmbeddingRuntime(configuration, options = {}) { const { model, modelDir } = await resolveEmbeddingModel(configuration, options); @@ -46,112 +33,43 @@ async function createEmbeddingRuntime(configuration, options = {}) { } async function resolveEmbeddingModel(configuration, options = {}) { - const { - root = process.cwd(), - env = process.env, - fetchImpl, - resolvePreset = resolveModelPreset, - warn = (message) => console.warn(message) - } = options; + const { root = process.cwd() } = options; const { model: modelName, directory } = normalizeEmbeddingConfiguration(configuration); + const modelDir = path.resolve(root, directory); - if (directory) { - const modelDir = path.resolve(root, directory); - - let model; - try { - model = await readModelLock(modelDir); - } catch (error) { - const recovery = /Embedding model lock not found/.test(error.message) - ? modelInstallHint(modelName, directory) - : `Remove or replace the invalid lock explicitly, then ${lowercaseFirst( - modelInstallHint(modelName, directory) - )}`; - throw new Error(`${error.message}. ${recovery}`, { cause: error }); - } - - const preset = findModelPreset(modelName); - if (preset && modelDescriptorDigest(model) !== modelDescriptorDigest(preset)) { - throw new Error( - `Embedding model directory ${modelDir} does not contain the pinned ${modelName} preset. Choose another directory or remove it explicitly before provisioning it again.` - ); - } - if (!preset && model.repository !== modelName) { - throw new Error( - `Embedding model directory ${modelDir} contains ${model.repository}, not ${modelName}. Choose another directory or remove it explicitly before provisioning it again.` - ); - } - try { - await verifyModelDirectory(modelDir, model); - } catch (error) { - throw new Error(`${error.message}. ${modelInstallHint(modelName, directory)}`, { - cause: error - }); - } - return { model, modelDir }; + let model; + try { + model = await readModelLock(modelDir); + } catch (error) { + const recovery = /Embedding model lock not found/.test(error.message) + ? modelInstallHint(directory) + : `Remove or replace the invalid lock explicitly, then ${lowercaseFirst( + modelInstallHint(directory) + )}`; + throw new Error(`${error.message}. ${recovery}`, { cause: error }); } - const model = resolvePreset(modelName); - const modelDir = getModelCacheDir(getModelCacheRoot(env), model); + if (model.repository !== modelName) { + throw new Error( + `Embedding model directory ${modelDir} contains ${model.repository}, not ${modelName}. Choose another directory or provision the configured model there.` + ); + } try { await verifyModelDirectory(modelDir, model); } catch (error) { - if (!/Embedding model is not provisioned or failed integrity checks/.test(error.message)) { - throw error; - } - warn( - `Embedding model '${model.repository}' is not available in '${modelDir}'. Downloading it now; application startup may be delayed. ${modelInstallHint(model.repository)}` - ); - try { - await provisionModelOnDemand(modelDir, model, { - env, - fetchImpl, - timeoutMs: options.provisionTimeoutMs, - retryMs: options.provisionRetryMs - }); - } catch (provisioningError) { - throw new Error( - `Failed to download embedding model '${model.repository}': ${provisioningError.message}. ${modelInstallHint(model.repository)}`, - { cause: provisioningError } - ); - } + throw new Error(`${error.message}. ${modelInstallHint(directory)}`, { + cause: error + }); } return { model, modelDir }; } -async function provisionModelOnDemand(modelDir, model, options) { - const timeoutMs = options.timeoutMs ?? MODEL_PROVISION_TIMEOUT_MS; - const retryMs = options.retryMs ?? MODEL_PROVISION_RETRY_MS; - const deadline = Date.now() + timeoutMs; - - while (true) { - try { - // eslint-disable-next-line no-await-in-loop - return await provisionModel(model, { - directory: modelDir, - env: options.env, - fetchImpl: options.fetchImpl - }); - } catch (error) { - if (error.code !== MODEL_PROVISIONING_IN_PROGRESS) throw error; - const remaining = deadline - Date.now(); - if (remaining <= 0) { - throw new Error(`Timed out waiting for embedding model provisioning in ${modelDir}`, { - cause: error - }); - } - // eslint-disable-next-line no-await-in-loop - await delay(Math.min(retryMs, remaining)); - } - } -} - function normalizeEmbeddingConfiguration(configuration) { if (configuration == null) { throw new Error('cds.env.requires.db.embedding.model must be a non-empty string'); } if (typeof configuration !== 'object' || Array.isArray(configuration)) { - throw new TypeError('embedding must be an object with model and optional directory'); + throw new TypeError('embedding must be an object with model and directory'); } const unsupported = Object.keys(configuration).filter( @@ -165,21 +83,14 @@ function normalizeEmbeddingConfiguration(configuration) { if (typeof configuration.model !== 'string' || !configuration.model.trim()) { throw new Error('cds.env.requires.db.embedding.model must be a non-empty string'); } - if ( - configuration.directory !== undefined && - (typeof configuration.directory !== 'string' || !configuration.directory.trim()) - ) { - throw new Error('embedding.directory must be a non-empty string'); + if (typeof configuration.directory !== 'string' || !configuration.directory.trim()) { + throw new Error('cds.env.requires.db.embedding.directory must be a non-empty string'); } return { model: configuration.model, directory: configuration.directory }; } -function modelInstallHint(repository, directory) { - const modelArgument = findModelPreset(repository) - ? repository - : '--descriptor '; - const directoryArgument = directory ? ` --directory ${directory}` : ''; - return `Run 'npx cds-ai model install ${modelArgument}${directoryArgument}'.`; +function modelInstallHint(directory) { + return `Run 'npx cds-ai model install --descriptor --directory ${directory}'.`; } function lowercaseFirst(value) { diff --git a/lib/vector_embedding/model-utils.js b/lib/vector_embedding/model-utils.js index c3a2991..e87c135 100644 --- a/lib/vector_embedding/model-utils.js +++ b/lib/vector_embedding/model-utils.js @@ -131,24 +131,6 @@ function fileForRole(model, role) { return model.files.find((file) => file.role === role); } -function artifactSetDigest(model) { - const files = model.files - .map(({ role, name, path: remotePath, size, sha256: checksum }) => ({ - role, - name, - path: remotePath, - size, - sha256: checksum - })) - .sort((left, right) => JSON.stringify(left).localeCompare(JSON.stringify(right))); - const canonical = JSON.stringify({ - repository: model.repository, - revision: model.revision.toLowerCase(), - files - }); - return createHash('sha256').update(canonical).digest('hex'); -} - function modelDescriptorDigest(model) { validateModelDescriptor(model); const files = model.files @@ -175,33 +157,6 @@ function modelDescriptorDigest(model) { return createHash('sha256').update(canonical).digest('hex'); } -function getModelCacheDir(cacheRoot, model) { - validateModelDescriptor(model); - return path.join( - cacheRoot, - ...model.repository.split('/'), - model.revision.toLowerCase(), - artifactSetDigest(model) - ); -} - -function getDataDir(appName = 'semantic-search', environment = process.env) { - const home = os.homedir(); - const directory = - os.platform() === 'win32' - ? environment.LOCALAPPDATA || environment.APPDATA || path.join(home, 'AppData', 'Local') - : environment.XDG_DATA_HOME || path.join(home, '.local', 'share'); - - return path.join(directory, appName); -} - -function getModelCacheRoot(environment = process.env) { - return ( - environment.CDS_AI_MODEL_CACHE || - path.join(getDataDir('semantic-search', environment), 'models') - ); -} - async function sha256(filePath) { const hash = createHash('sha256'); for await (const chunk of createReadStream(filePath)) hash.update(chunk); @@ -517,9 +472,10 @@ async function writeModelLock(modelDir, model) { async function provisionModel(model, options = {}) { validateModelDescriptor(model); - const requestedDirectory = path.resolve( - options.directory ?? getModelCacheDir(getModelCacheRoot(options.env), model) - ); + if (typeof options.directory !== 'string' || !options.directory.trim()) { + throw new Error('A non-empty provisioning directory is required'); + } + const requestedDirectory = path.resolve(options.directory); const directory = await canonicalizeProvisioningDirectory(requestedDirectory); await ensureParentDirectories(path.dirname(directory)); @@ -760,13 +716,9 @@ async function loadModelAndTokenizer(modelDir, model) { export { MODEL_LOCK_FILE, - MODEL_PROVISIONING_IN_PROGRESS, downloadFile, downloadModelIfNeeded, - artifactSetDigest, fileForRole, - getModelCacheDir, - getModelCacheRoot, isValidFile, loadModelAndTokenizer, modelDescriptorDigest, diff --git a/lib/vector_embedding/models.js b/lib/vector_embedding/models.js deleted file mode 100644 index a33d47e..0000000 --- a/lib/vector_embedding/models.js +++ /dev/null @@ -1,52 +0,0 @@ -const MINILM_MODEL = Object.freeze({ - repository: 'Xenova/all-MiniLM-L6-v2', - revision: '751bff37182d3f1213fa05d7196b954e230abad9', - dimensions: 384, - maxLength: 128, - files: Object.freeze([ - Object.freeze({ - role: 'model', - name: 'model.onnx', - path: 'onnx/model.onnx', - size: 90387606, - sha256: '759c3cd2b7fe7e93933ad23c4c9181b7396442a2ed746ec7c1d46192c469c46e' - }), - Object.freeze({ - role: 'tokenizer', - name: 'tokenizer.json', - path: 'tokenizer.json', - size: 711661, - sha256: 'da0e79933b9ed51798a3ae27893d3c5fa4a201126cef75586296df9b4d2c62a0' - }), - Object.freeze({ - role: 'tokenizerConfig', - name: 'tokenizer_config.json', - path: 'tokenizer_config.json', - size: 366, - sha256: '9261e7d79b44c8195c1cada2b453e55b00aeb81e907a6664974b4d7776172ab3' - }) - ]), - output: Object.freeze({ - name: 'last_hidden_state', - pooling: 'mean', - normalize: true - }) -}); - -const MODEL_PRESETS = new Map([[MINILM_MODEL.repository, MINILM_MODEL]]); - -function findModelPreset(name) { - return MODEL_PRESETS.get(name); -} - -function resolveModelPreset(name) { - const model = findModelPreset(name); - if (!model) { - throw new Error( - `Unsupported embedding model '${name}' for ad-hoc download. Provision a compatible custom model with --descriptor and --directory.` - ); - } - return model; -} - -export { MINILM_MODEL, findModelPreset, resolveModelPreset }; diff --git a/package.json b/package.json index 16b00e2..f38cc49 100644 --- a/package.json +++ b/package.json @@ -13,9 +13,9 @@ }, "scripts": { "lint": "npx -y eslint@10 .", - "model:provision": "node bin/cds-ai.js model install Xenova/all-MiniLM-L6-v2", - "pretest": "npm run model:provision", - "pretest:hybrid": "npm run model:provision", + "test:model:provision": "node bin/cds-ai.js model install --descriptor tests/fixtures/minilm.embedding-model.json --directory tests/.models/minilm", + "pretest": "npm run test:model:provision", + "pretest:hybrid": "npm run test:model:provision", "test": "node --test tests/*.test.js", "test:hybrid": "cds bind --exec -- node --test tests/*.test.js tests/integration/*.test.js", "format": "npx -y prettier@3 . --write && format-cds -f", diff --git a/tests/fixtures/minilm.embedding-model.json b/tests/fixtures/minilm.embedding-model.json new file mode 100644 index 0000000..2185c0b --- /dev/null +++ b/tests/fixtures/minilm.embedding-model.json @@ -0,0 +1,34 @@ +{ + "repository": "Xenova/all-MiniLM-L6-v2", + "revision": "751bff37182d3f1213fa05d7196b954e230abad9", + "dimensions": 384, + "maxLength": 128, + "files": [ + { + "role": "model", + "name": "model.onnx", + "path": "onnx/model.onnx", + "size": 90387606, + "sha256": "759c3cd2b7fe7e93933ad23c4c9181b7396442a2ed746ec7c1d46192c469c46e" + }, + { + "role": "tokenizer", + "name": "tokenizer.json", + "path": "tokenizer.json", + "size": 711661, + "sha256": "da0e79933b9ed51798a3ae27893d3c5fa4a201126cef75586296df9b4d2c62a0" + }, + { + "role": "tokenizerConfig", + "name": "tokenizer_config.json", + "path": "tokenizer_config.json", + "size": 366, + "sha256": "9261e7d79b44c8195c1cada2b453e55b00aeb81e907a6664974b4d7776172ab3" + } + ], + "output": { + "name": "last_hidden_state", + "pooling": "mean", + "normalize": true + } +} diff --git a/tests/model-provisioning.test.js b/tests/model-provisioning.test.js index af74279..f5db013 100644 --- a/tests/model-provisioning.test.js +++ b/tests/model-provisioning.test.js @@ -8,12 +8,10 @@ import { runModelCommand } from '../lib/vector_embedding/cli.js'; import { resolveEmbeddingModel } from '../lib/vector_embedding/embedding.js'; import { MODEL_LOCK_FILE, - getModelCacheDir, provisionModel, readModelLock, verifyModelDirectory } from '../lib/vector_embedding/model-utils.js'; -import { MINILM_MODEL, resolveModelPreset } from '../lib/vector_embedding/models.js'; const temporaryDirectories = []; @@ -25,16 +23,9 @@ afterEach(async () => { ); }); -describe('model presets', () => { - test('resolves the MiniLM model preset by repository name', () => { - assert.equal(resolveModelPreset('Xenova/all-MiniLM-L6-v2'), MINILM_MODEL); - }); - - test('rejects unknown model names', () => { - assert.throws(() => resolveModelPreset('example/unknown'), /Unsupported embedding model/); - }); - +describe('runtime model configuration', () => { test('accepts only model and directory in runtime configuration', async () => { + const model = fixtureModel(Buffer.from('configuration fixture')); await assert.rejects( resolveEmbeddingModel(), /cds\.env\.requires\.db\.embedding\.model must be a non-empty string/ @@ -44,16 +35,20 @@ describe('model presets', () => { /cds\.env\.requires\.db\.embedding\.model must be a non-empty string/ ); await assert.rejects( - resolveEmbeddingModel(MINILM_MODEL.repository), - /embedding must be an object with model and optional directory/ + resolveEmbeddingModel(model.repository), + /embedding must be an object with model and directory/ ); await assert.rejects( - resolveEmbeddingModel({ ...MINILM_MODEL }), + resolveEmbeddingModel({ ...model }), /Only model and directory are supported/ ); await assert.rejects( - resolveEmbeddingModel({ model: MINILM_MODEL.repository, directory: '' }), - /embedding.directory must be a non-empty string/ + resolveEmbeddingModel({ model: model.repository }), + /cds\.env\.requires\.db\.embedding\.directory must be a non-empty string/ + ); + await assert.rejects( + resolveEmbeddingModel({ model: model.repository, directory: '' }), + /cds\.env\.requires\.db\.embedding\.directory must be a non-empty string/ ); }); }); @@ -76,6 +71,10 @@ describe('explicit model provisioning', () => { ) ); assert.deepEqual(await readModelLock(directory), model); + assert.deepEqual(JSON.parse(await fs.readFile(path.join(directory, MODEL_LOCK_FILE), 'utf8')), { + ...model, + formatVersion: 1 + }); assert.deepEqual((await fs.readdir(directory)).sort(), [ MODEL_LOCK_FILE, 'model.onnx', @@ -318,92 +317,13 @@ describe('explicit model provisioning', () => { assert.deepEqual(await fs.readdir(directory), []); }); - test('downloads a missing model into the automatic cache and reuses it', async () => { - const cacheRoot = await createTemporaryDirectory(); - const content = Buffer.from('lazy download fixture'); - const model = fixtureModel(content); - const requestedUrls = []; - const warnings = []; - const options = { - env: { CDS_AI_MODEL_CACHE: cacheRoot }, - fetchImpl: createFetch(content, requestedUrls), - resolvePreset(name) { - assert.equal(name, model.repository); - return model; - }, - warn: (message) => warnings.push(message) - }; - - const first = await resolveEmbeddingModel({ model: model.repository }, options); - const expectedDirectory = getModelCacheDir(cacheRoot, model); - - assert.equal(first.model, model); - assert.equal(first.modelDir, expectedDirectory); - assert.equal(warnings.length, 1); - assert.match(warnings[0], /Downloading it now; application startup may be delayed/); - assert.equal(requestedUrls.length, model.files.length); - assert.deepEqual(await readModelLock(expectedDirectory), model); - - const second = await resolveEmbeddingModel({ model: model.repository }, options); - assert.equal(second.modelDir, expectedDirectory); - assert.equal(warnings.length, 1); - assert.equal(requestedUrls.length, model.files.length); - }); - - test('waits for concurrent lazy provisioning and reuses the completed download', async () => { - const cacheRoot = await createTemporaryDirectory(); - const content = Buffer.from('concurrent lazy download fixture'); - const model = fixtureModel(content); - const requestedUrls = []; - const warnings = []; - let releaseDownload; - let signalDownloadStarted; - let firstRequest = true; - const downloadStarted = new Promise((resolve) => { - signalDownloadStarted = resolve; - }); - const waitForRelease = new Promise((resolve) => { - releaseDownload = resolve; - }); - const fetchImpl = async (url) => { - requestedUrls.push(url); - if (firstRequest) { - firstRequest = false; - signalDownloadStarted(); - await waitForRelease; - } - return new Response(content, { - headers: { 'content-length': String(content.length) } - }); - }; - const options = { - env: { CDS_AI_MODEL_CACHE: cacheRoot }, - fetchImpl, - resolvePreset: () => model, - warn: (message) => warnings.push(message), - provisionRetryMs: 5, - provisionTimeoutMs: 1000 - }; - - const first = resolveEmbeddingModel({ model: model.repository }, options); - await downloadStarted; - const second = resolveEmbeddingModel({ model: model.repository }, options); - await new Promise((resolve) => setTimeout(resolve, 20)); - releaseDownload(); - - const resolved = await Promise.all([first, second]); - assert.equal(resolved[0].modelDir, resolved[1].modelDir); - assert.equal(warnings.length, 2); - assert.equal(requestedUrls.length, model.files.length); - }); - test('keeps explicitly configured directories offline', async () => { const root = await createTemporaryDirectory(); let fetched = false; await assert.rejects( resolveEmbeddingModel( { - model: MINILM_MODEL.repository, + model: 'example/model', directory: './models/minilm' }, { @@ -414,7 +334,7 @@ describe('explicit model provisioning', () => { } } ), - /cds-ai model install Xenova\/all-MiniLM-L6-v2 --directory \.\/models\/minilm/ + /cds-ai model install --descriptor --directory \.\/models\/minilm/ ); assert.equal(fetched, false); }); @@ -439,22 +359,19 @@ describe('explicit model provisioning', () => { assert.equal(absolute.modelDir, directory); }); - test('requires the complete built-in preset in an explicitly configured directory', async () => { + test('rejects a configured model name that does not match the provisioned lock', async () => { const directory = await createTemporaryDirectory(); - const content = Buffer.from('imposter preset fixture'); - const model = { - ...fixtureModel(content), - repository: MINILM_MODEL.repository - }; + const content = Buffer.from('repository mismatch fixture'); + const model = fixtureModel(content); await provisionModel(model, { directory, fetchImpl: createFetch(content) }); await assert.rejects( - resolveEmbeddingModel({ model: MINILM_MODEL.repository, directory }), - /does not contain the pinned Xenova\/all-MiniLM-L6-v2 preset/ + resolveEmbeddingModel({ model: 'example/other-model', directory }), + /contains example\/model, not example\/other-model/ ); }); - test('gives custom models a descriptor-based provisioning command', async () => { + test('gives models a descriptor-based provisioning command', async () => { const root = await createTemporaryDirectory(); await assert.rejects( @@ -468,7 +385,7 @@ describe('explicit model provisioning', () => { await fs.writeFile(path.join(directory, MODEL_LOCK_FILE), '{}'); await assert.rejects( - resolveEmbeddingModel({ model: MINILM_MODEL.repository, directory }), + resolveEmbeddingModel({ model: 'example/model', directory }), /Remove or replace the invalid lock explicitly, then run 'npx cds-ai model install/ ); }); @@ -495,10 +412,22 @@ describe('explicit model provisioning', () => { assert.match(output.join(''), /Provisioned example\/model/); }); - test('requires a directory when provisioning a custom descriptor', async () => { + test('requires a descriptor and directory when provisioning', async () => { + await assert.rejects( + provisionModel(fixtureModel(Buffer.from('missing directory fixture'))), + /non-empty provisioning directory is required/ + ); await assert.rejects( runModelCommand(['model', 'install', '--descriptor', './embedding-model.json']), - /--descriptor requires --directory/ + /--directory is required/ + ); + await assert.rejects( + runModelCommand(['model', 'install', '--directory', './models/custom']), + /--descriptor is required/ + ); + await assert.rejects( + runModelCommand(['model', 'install', 'example/model', '--directory', './models/custom']), + /Unexpected argument 'example\/model'/ ); }); }); diff --git a/tests/vector-unit.test.js b/tests/vector-unit.test.js index bb6f6d6..a59a59d 100644 --- a/tests/vector-unit.test.js +++ b/tests/vector-unit.test.js @@ -11,10 +11,8 @@ import { tokenizeWithChunks } from '../lib/vector_embedding/embedding.js'; import { - artifactSetDigest, downloadFile, downloadModelIfNeeded, - getModelCacheDir, validateModelDescriptor } from '../lib/vector_embedding/model-utils.js'; @@ -167,31 +165,6 @@ describe('model compatibility', () => { }); }); -describe('model cache identity', () => { - test('preserves repository components and separates artifact variants', () => { - const root = path.join(path.sep, 'cache'); - const model = fixtureModel(Buffer.from('fixture')); - const reordered = { ...model, files: [...model.files].reverse() }; - const variant = { - ...model, - files: model.files.map((file, index) => - index === 0 ? { ...file, sha256: 'f'.repeat(64) } : file - ) - }; - const flattenedRepository = { ...model, repository: 'example_model' }; - - const modelPath = getModelCacheDir(root, model); - assert.equal(modelPath, getModelCacheDir(root, reordered)); - assert.notEqual(modelPath, getModelCacheDir(root, variant)); - assert.notEqual(modelPath, getModelCacheDir(root, flattenedRepository)); - assert.equal( - path.relative(root, modelPath).split(path.sep).slice(0, 3).join('/'), - `example/model/${model.revision}` - ); - assert.equal(path.basename(modelPath), artifactSetDigest(model)); - }); -}); - describe('model download', () => { test('uses a pinned revision and atomically caches verified files', async () => { const directory = await createTemporaryDirectory(); diff --git a/tests/vector.test.js b/tests/vector.test.js index 3f5223d..a0cd4b8 100644 --- a/tests/vector.test.js +++ b/tests/vector.test.js @@ -1,13 +1,22 @@ import { after, before, describe, test } from 'node:test'; import assert from 'node:assert'; +import fs from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; import cds from '@sap/cds'; import { initializeEmbedding, vector_embedding } from '../lib/vector_embedding/index.js'; -import { MINILM_MODEL } from '../lib/vector_embedding/models.js'; + +const MINILM_MODEL = JSON.parse( + await fs.readFile(new URL('./fixtures/minilm.embedding-model.json', import.meta.url), 'utf8') +); +const MINILM_DIRECTORY = fileURLToPath(new URL('./.models/minilm', import.meta.url)); let embeddingModule; before(async () => { - embeddingModule = await initializeEmbedding({ model: MINILM_MODEL.repository }); + embeddingModule = await initializeEmbedding({ + model: MINILM_MODEL.repository, + directory: MINILM_DIRECTORY + }); }); describe('Vector embedding function (standalone)', () => { @@ -143,11 +152,23 @@ describe('ai-sqlite integration', () => { ); }); + test('requires an explicitly configured embedding directory during startup', async () => { + await assert.rejects( + cds.connect.to('missing-embedding-directory-db', { + kind: 'ai-sqlite', + embedding: { model: MINILM_MODEL.repository }, + credentials: { url: ':memory:' } + }), + /cds\.env\.requires\.db\.embedding\.directory must be a non-empty string/ + ); + }); + before(async () => { db = await cds.connect.to('vector-db', { kind: 'ai-sqlite', embedding: { - model: MINILM_MODEL.repository + model: MINILM_MODEL.repository, + directory: MINILM_DIRECTORY }, credentials: { url: ':memory:' } }); From a9ec543b231df75218a93510d8f0c3bc17ded8e2 Mon Sep 17 00:00:00 2001 From: Sebastian Van Syckel Date: Thu, 27 Aug 2026 12:49:32 +0200 Subject: [PATCH 6/6] feat: provision embedding models by name --- .gitignore | 2 +- CHANGELOG.md | 5 +- README.md | 122 +++---- lib/sqlite/AISQLiteService.js | 25 +- lib/vector_embedding/InferenceSession.js | 40 ++- lib/vector_embedding/cli.js | 59 ++-- lib/vector_embedding/embedding.js | 149 +++++--- lib/vector_embedding/model-discovery.js | 390 +++++++++++++++++++++ lib/vector_embedding/model-install.js | 64 ++++ lib/vector_embedding/model-utils.js | 21 +- package.json | 2 +- tests/fixtures/minilm.embedding-model.json | 34 -- tests/model-discovery.test.js | 326 +++++++++++++++++ tests/model-provisioning.test.js | 216 +++++++++--- tests/vector-unit.test.js | 39 +++ tests/vector.test.js | 30 +- 16 files changed, 1241 insertions(+), 283 deletions(-) create mode 100644 lib/vector_embedding/model-discovery.js create mode 100644 lib/vector_embedding/model-install.js delete mode 100644 tests/fixtures/minilm.embedding-model.json create mode 100644 tests/model-discovery.test.js diff --git a/.gitignore b/.gitignore index 2a91514..2ce6852 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,4 @@ package-lock.json .env .cdsrc-private.json resources/ -tests/.models/ +.cds/models/ diff --git a/CHANGELOG.md b/CHANGELOG.md index f93e39c..36480ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,10 @@ ### Added - Add the `ai-sqlite` kind with a `VECTOR_EMBEDDING` function using compatible ONNX encoder models - - Requires `cds.env.requires.db.embedding.model` and `directory`; adds `cds-ai model install` for explicit, checksum-verified model provisioning from a descriptor + - Requires `cds.env.requires.db.embedding.model`; automatically discovers model metadata and supports warned, on-demand provisioning into `.cds/models` + - Adds `npx @cap-js/ai install-model ` with an optional shared model-cache root - Uses `@huggingface/tokenizers` and chunks long input without dropping per-chunk special tokens - - Configures embedding runtimes only through `model` and a relative or absolute `directory`; model metadata remains in the provisioned lock + - Configures embedding runtimes only through `model` and an optional relative, absolute, or home-relative `directory`; discovered metadata remains in the provisioned lock - Supports both 3-parameter `(text, text_type, model_and_version)` and 4-parameter variants with `remote_source` - Compatible with `SAP_GXY.20250407` and `SAP_GXY.20240715` model versions - Synchronous execution suitable for SQLite user-defined functions diff --git a/README.md b/README.md index 180d257..a7daa3f 100644 --- a/README.md +++ b/README.md @@ -221,7 +221,7 @@ npm add @cap-js/sqlite onnxruntime-node@1.20.1 #### Model provisioning -Runtime configuration is intentionally limited to a model name and directory: +Runtime configuration is intentionally limited to a model name and an optional model-cache root: ```json { @@ -230,8 +230,7 @@ Runtime configuration is intentionally limited to a model name and directory: "db": { "kind": "ai-sqlite", "embedding": { - "model": "organization/model", - "directory": "./models/embedding" + "model": "foo/bar" } } } @@ -239,25 +238,47 @@ Runtime configuration is intentionally limited to a model name and directory: } ``` -Both `embedding.model` and `embedding.directory` are required. `ai-sqlite` fails during startup when either is absent. No revision, dimensions, tokenizer, file, pooling, or checksum settings are accepted in runtime configuration. +`embedding.model` is required. If it is absent, `ai-sqlite` fails during startup. No revision, dimensions, tokenizer, file, pooling, checksum, or descriptor settings are accepted in runtime configuration. -Create an `embedding-model.json` descriptor for the model, then provision it before application startup: +Without `directory`, the model is stored below the CAP project at `.cds/models/foo/bar`. Startup reuses a valid installation from there. If it is missing, startup logs a warning, discovers and downloads the model, generates `embedding.lock.json`, and reuses that installation on subsequent starts. + +To provision the project-local model before startup instead: ```sh -npx cds-ai model install --descriptor ./embedding-model.json --directory ./models/embedding +npx @cap-js/ai install-model foo/bar ``` -Provisioning downloads the descriptor's pinned artifacts, verifies their sizes and SHA-256 checksums, and writes the validated descriptor to `embedding.lock.json`. The lock content is therefore generated from the descriptor; it is not inferred from the model name. +To share a model across projects, select another cache root: -The CLI resolves a relative `--directory` from its working directory. Runtime configuration resolves a relative `embedding.directory` from `cds.root`; absolute directories are used unchanged in both cases. Run the install command from `cds.root` or use the same absolute path so both refer to the same model directory. +```sh +npx @cap-js/ai install-model foo/bar --directory ~/.cds/models +``` -A configured directory must already contain a valid `embedding.lock.json` and all verified artifacts. Startup remains offline and fails rather than downloading or modifying the directory if it is incomplete. +```json +{ + "cds": { + "requires": { + "db": { + "kind": "ai-sqlite", + "embedding": { + "model": "foo/bar", + "directory": "~/.cds/models" + } + } + } + } +} +``` + +`directory` always names the cache root; the model is stored below it using the repository path, for example `~/.cds/models/foo/bar`. Relative directories are resolved from `cds.root`, absolute directories are used unchanged, and `~/` is resolved from the user's home directory. + +When `directory` is configured, startup treats it as a pre-installed shared cache: it verifies the model but does not download or modify it. This makes runtime deployment deterministic and allows the shared directory to be read-only. -##### Model metadata +##### Automatic model discovery -At startup, the runtime reads the immutable revision, artifacts, checksums, dimensions, tokenizer limit, pooling, and normalization from `embedding.lock.json`. It verifies the files and checks that the lock's repository matches `embedding.model`. +The installer resolves the model's current Hugging Face revision to an immutable commit, selects the conventional `onnx/model.onnx` and tokenizer/configuration files, calculates or obtains their checksums, and derives the dimensions, tokenizer limit, pooling, and normalization metadata. It then writes all resolved metadata to `embedding.lock.json` alongside the downloaded artifacts. -This is local metadata detection, not discovery from a model name or arbitrary Hugging Face repository. A model name alone cannot reliably determine artifact selection, pooling, normalization, or other runtime semantics. +Discovery supports compatible Hugging Face ONNX Sentence Transformers models with machine-readable pooling semantics. Repositories with missing or ambiguous artifacts or semantics fail with a compatibility error instead of using guessed defaults. Once installed, startup uses the pinned lock and does not follow later changes to the model repository. The HANA-compatible SQL function can then be used in CQL: @@ -280,87 +301,26 @@ SELECT.from('Books').columns` **Features:** - **Initialization**: The ONNX model is loaded when the `ai-sqlite` service starts -- **Explicit provisioning**: Install models before startup with `cds-ai model install` +- **Automatic provisioning**: Use model-only configuration for warned, on-demand installation into `.cds/models` +- **Explicit provisioning**: Preinstall local or shared models with `npx @cap-js/ai install-model` - **Verified artifacts**: The provisioned lock pins the revision, artifact sizes, and SHA-256 checksums - **Hugging Face tokenization**: Uses `@huggingface/tokenizers` and safely chunks text that exceeds the model limit - **Deterministic**: Same input always produces same output -- **Configurable output handling**: The descriptor controls pooling and L2 normalization +- **Automatic output handling**: Pooling and normalization are derived from Sentence Transformers metadata - **Semantic similarity**: Embeddings capture text meaning for similarity search #### Compatible encoder models -Create an `embedding-model.json` descriptor for each compatible model. Models are not discovered dynamically: every artifact must belong to an immutable revision and have an expected size and SHA-256 checksum. - -```json -{ - "repository": "organization/model", - "revision": "0123456789abcdef0123456789abcdef01234567", - "dimensions": 768, - "maxLength": 512, - "files": [ - { - "role": "model", - "name": "model.onnx", - "path": "onnx/model.onnx", - "size": 123456789, - "sha256": "<64 lowercase hexadecimal characters>" - }, - { - "role": "tokenizer", - "name": "tokenizer.json", - "path": "tokenizer.json", - "size": 123456, - "sha256": "<64 lowercase hexadecimal characters>" - }, - { - "role": "tokenizerConfig", - "name": "tokenizer_config.json", - "path": "tokenizer_config.json", - "size": 1234, - "sha256": "<64 lowercase hexadecimal characters>" - } - ], - "output": { - "name": "last_hidden_state", - "pooling": "mean", - "normalize": true - } -} -``` - -Provision it into an application-managed directory: - -```sh -npx cds-ai model install --descriptor ./embedding-model.json --directory ./models/custom -``` - -Then point the service at that locked directory. Runtime configuration contains only the model name and directory; the lock contains the technical model metadata: - -```json -{ - "cds": { - "requires": { - "db": { - "kind": "ai-sqlite", - "embedding": { - "model": "organization/model", - "directory": "./models/custom" - } - } - } - } -} -``` - -Provisioning canonicalizes symlinked parent directories and rejects a model directory that is itself a symlink. +Compatible repositories must provide `onnx/model.onnx`, `tokenizer.json`, `tokenizer_config.json`, and `config.json`. The model must accept `input_ids` and may additionally accept `attention_mask` and `token_type_ids`, all as `int64` tensors, and expose a `last_hidden_state` float output. -The runtime accepts no model metadata beyond `embedding.model` and `embedding.directory`. Every model must first be installed from a descriptor into the configured directory. +Pooling semantics are read from Sentence Transformers `modules.json` and its pooling configuration. Converted repositories such as `Xenova/*` can declare a single `base_model`; its immutable Sentence Transformers metadata is used to determine mean or CLS pooling and normalization. Unsupported module chains, ambiguous pooling modes, missing metadata, or incompatible ONNX inputs and outputs fail explicitly. -Compatible models must accept `input_ids` and may additionally accept `attention_mask` and `token_type_ids`, all as `int64` tensors. Their configured float32 or float64 output must support `mean` or `cls` pooling from `[1, sequence, dimensions]`, or `none` for an already pooled `[dimensions]` or `[1, dimensions]` tensor. Additional pinned ONNX data files can use the `auxiliary` role. Startup probes the model and rejects incompatible input names, output names, types, shapes, or dimensions. +Provisioning canonicalizes symlinked parent directories and rejects a model directory that is itself a symlink. Existing valid locks remain pinned and are reused rather than silently following changes to the repository's default branch. **Error Handling:** -- Starting `ai-sqlite` fails if `cds.env.requires.db.embedding.model` or `cds.env.requires.db.embedding.directory` is not set, or the ONNX model cannot be initialized +- Starting `ai-sqlite` fails if `cds.env.requires.db.embedding.model` is not set or the ONNX model cannot be initialized +- A missing model in the project-local `.cds/models` cache is installed after a startup warning - Starting `ai-sqlite` fails with a provisioning command if a configured model directory is missing or fails integrity checks - Provisioning downloads are time-limited and accepted only when their expected size and SHA-256 match - Throws if embedding generation fails diff --git a/lib/sqlite/AISQLiteService.js b/lib/sqlite/AISQLiteService.js index 07212ac..498ba47 100644 --- a/lib/sqlite/AISQLiteService.js +++ b/lib/sqlite/AISQLiteService.js @@ -7,10 +7,29 @@ const LOG = cds.log('@cap-js/ai'); export default class AISQLiteService extends SQLiteService { async init() { this._embeddingRuntime = await createEmbeddingRuntime(this.options.embedding, { - root: cds.root + root: cds.root, + warn: (message) => LOG.warn(message) }); - LOG.info('Vector embedding ONNX model initialized'); - return super.init(); + try { + const service = await super.init(); + LOG.info('Vector embedding ONNX model initialized'); + return service; + } catch (error) { + await this._embeddingRuntime.dispose().catch(() => {}); + this._embeddingRuntime = undefined; + throw error; + } + } + + async disconnect(tenant) { + try { + return await super.disconnect(tenant); + } finally { + if (tenant === undefined) { + await this._embeddingRuntime?.dispose(); + this._embeddingRuntime = undefined; + } + } } get factory() { diff --git a/lib/vector_embedding/InferenceSession.js b/lib/vector_embedding/InferenceSession.js index 45ac693..ecb61fd 100644 --- a/lib/vector_embedding/InferenceSession.js +++ b/lib/vector_embedding/InferenceSession.js @@ -62,6 +62,13 @@ class InferenceSession { return output; } + dispose() { + const handler = this.handler; + if (!handler) return; + this.handler = undefined; + return handler.dispose(); + } + static async create(pathOrBuffer) { if (typeof pathOrBuffer !== 'string' && !(pathOrBuffer instanceof Uint8Array)) { throw new TypeError('Expected an ONNX model path or Uint8Array'); @@ -73,25 +80,34 @@ class InferenceSession { class SynchronousSessionHandler { constructor(pathOrBuffer) { this.session = new binding.InferenceSession(); - if (typeof pathOrBuffer === 'string') { - this.session.loadModel(pathOrBuffer, {}); - } else { - this.session.loadModel( - pathOrBuffer.buffer, - pathOrBuffer.byteOffset, - pathOrBuffer.byteLength, - {} - ); + try { + if (typeof pathOrBuffer === 'string') { + this.session.loadModel(pathOrBuffer, {}); + } else { + this.session.loadModel( + pathOrBuffer.buffer, + pathOrBuffer.byteOffset, + pathOrBuffer.byteLength, + {} + ); + } + this.inputNames = this.session.inputNames; + this.outputNames = this.session.outputNames; + } catch (error) { + try { + this.session.dispose(); + } catch { + // Preserve the model loading error. + } + throw error; } - this.inputNames = this.session.inputNames; - this.outputNames = this.session.outputNames; } run(feeds, fetches, options) { return this.session.run(feeds, fetches, options); } - async dispose() { + dispose() { this.session.dispose(); } } diff --git a/lib/vector_embedding/cli.js b/lib/vector_embedding/cli.js index 0b162a2..956c0ed 100644 --- a/lib/vector_embedding/cli.js +++ b/lib/vector_embedding/cli.js @@ -1,67 +1,58 @@ -import fs from 'node:fs/promises'; -import path from 'node:path'; -import { provisionModel, validateModelDescriptor } from './model-utils.js'; +import { installModel } from './model-install.js'; +import { validateEmbeddingModel } from './embedding.js'; const HELP = `Usage: - cds-ai model install --descriptor --directory + npx @cap-js/ai install-model [--directory ] Options: - --directory Install into an application-managed directory - --descriptor Read a compatible model descriptor from JSON + --directory Use this model-cache root instead of .cds/models --help Show this help `; async function runModelCommand(argv, options = {}) { - const { cwd = process.cwd(), fetchImpl, stdout = process.stdout } = options; + const { cwd = process.cwd(), stdout = process.stdout } = options; const command = parseArguments(argv); if (command.help) { stdout.write(HELP); return; } - const model = await readDescriptor(path.resolve(cwd, command.descriptor)); - const directory = path.resolve(cwd, command.directory); - - await provisionModel(model, { directory, fetchImpl }); - stdout.write(`Provisioned ${model.repository} in ${directory}\n`); + const { modelDir } = await installModel(command.model, { + root: cwd, + directory: command.directory, + home: options.home, + fetchImpl: options.fetchImpl, + discover: options.discover, + validate: options.validate ?? validateEmbeddingModel, + timeoutMs: options.timeoutMs, + retryMs: options.retryMs + }); + stdout.write(`Installed ${command.model} in ${modelDir}\n`); } function parseArguments(argv) { if (argv.length === 0 || argv.includes('--help') || argv.includes('-h')) return { help: true }; - if (argv[0] !== 'model' || argv[1] !== 'install') { + if (argv[0] !== 'install-model') { throw new Error(`Unsupported command.\n\n${HELP}`); } - let descriptor; + let model; let directory; - for (let index = 2; index < argv.length; index++) { + for (let index = 1; index < argv.length; index++) { const argument = argv[index]; - if (argument === '--descriptor' || argument === '--directory') { + if (argument === '--directory') { const value = argv[++index]; if (!value || value.startsWith('--')) throw new Error(`${argument} requires a value`); - if (argument === '--descriptor') descriptor = value; - else directory = value; + directory = value; continue; } if (argument.startsWith('-')) throw new Error(`Unknown option '${argument}'`); - throw new Error(`Unexpected argument '${argument}'`); + if (model) throw new Error(`Unexpected argument '${argument}'`); + model = argument; } - if (!descriptor) throw new Error('--descriptor is required'); - if (!directory) throw new Error('--directory is required'); - return { descriptor, directory }; -} - -async function readDescriptor(file) { - let descriptor; - try { - descriptor = JSON.parse(await fs.readFile(file, 'utf8')); - } catch (error) { - throw new Error(`Cannot read embedding model descriptor at ${file}: ${error.message}`, { - cause: error - }); - } - return validateModelDescriptor(descriptor); + if (!model) throw new Error('Specify a model name'); + return { directory, model }; } export { HELP, parseArguments, runModelCommand }; diff --git a/lib/vector_embedding/embedding.js b/lib/vector_embedding/embedding.js index c42a153..63234b1 100644 --- a/lib/vector_embedding/embedding.js +++ b/lib/vector_embedding/embedding.js @@ -1,52 +1,100 @@ -import path from 'path'; import { Tensor } from './InferenceSession.js'; -import { loadModelAndTokenizer, readModelLock, verifyModelDirectory } from './model-utils.js'; +import { installModel } from './model-install.js'; +import { + getModelDirectory, + getModelRoot, + loadModelAndTokenizer, + readModelLock, + verifyModelDirectory +} from './model-utils.js'; const STANDARD_INPUT_NAMES = new Set(['input_ids', 'attention_mask', 'token_type_ids']); async function createEmbeddingRuntime(configuration, options = {}) { const { model, modelDir } = await resolveEmbeddingModel(configuration, options); + return createEmbeddingRuntimeFromModel(modelDir, model); +} + +async function createEmbeddingRuntimeFromModel(modelDir, model) { const { session, tokenizer } = await loadModelAndTokenizer(modelDir, model); - const tokenizerState = createTokenizerState(tokenizer, model.maxLength); - - validateSession(session, model); - - const runtime = { - dimensions: model.dimensions, - embedding(text) { - const chunks = tokenizeWithChunks(String(text), tokenizer, tokenizerState); - return processChunkedEmbeddings(chunks, session, model); - }, - vectorEmbedding(text) { - if (!text) return JSON.stringify(new Array(model.dimensions).fill(0)); - return JSON.stringify(Array.from(this.embedding(text))); - } + let disposed = false; + const dispose = async () => { + if (disposed) return; + disposed = true; + await session.dispose(); }; - const probe = runtime.embedding('embedding model startup probe'); - if (probe.length !== model.dimensions) { - throw new Error( - `Embedding model produced ${probe.length} dimensions; configured ${model.dimensions}` - ); + try { + const tokenizerState = createTokenizerState(tokenizer, model.maxLength); + + validateSession(session, model); + + const runtime = { + dimensions: model.dimensions, + embedding(text) { + const chunks = tokenizeWithChunks(String(text), tokenizer, tokenizerState); + return processChunkedEmbeddings(chunks, session, model); + }, + vectorEmbedding(text) { + if (!text) return JSON.stringify(new Array(model.dimensions).fill(0)); + return JSON.stringify(Array.from(this.embedding(text))); + }, + dispose + }; + + const probe = runtime.embedding('embedding model startup probe'); + if (probe.length !== model.dimensions) { + throw new Error( + `Embedding model produced ${probe.length} dimensions; configured ${model.dimensions}` + ); + } + return runtime; + } catch (error) { + await dispose().catch(() => {}); + throw error; } - return runtime; +} + +async function validateEmbeddingModel(modelDir, model) { + const runtime = await createEmbeddingRuntimeFromModel(modelDir, model); + await runtime.dispose(); } async function resolveEmbeddingModel(configuration, options = {}) { - const { root = process.cwd() } = options; + const { + root = process.cwd(), + warn = (message) => console.warn(message), + fetchImpl, + discover, + validate + } = options; const { model: modelName, directory } = normalizeEmbeddingConfiguration(configuration); - const modelDir = path.resolve(root, directory); + const modelRoot = getModelRoot(directory, root, options.home); + const modelDir = getModelDirectory(modelRoot, modelName); + const installOptions = { + root, + directory: modelRoot, + home: options.home, + fetchImpl, + discover, + validate: validate ?? validateEmbeddingModel, + timeoutMs: options.provisionTimeoutMs, + retryMs: options.provisionRetryMs + }; let model; try { model = await readModelLock(modelDir); } catch (error) { - const recovery = /Embedding model lock not found/.test(error.message) - ? modelInstallHint(directory) - : `Remove or replace the invalid lock explicitly, then ${lowercaseFirst( - modelInstallHint(directory) - )}`; - throw new Error(`${error.message}. ${recovery}`, { cause: error }); + if (directory !== undefined || !/Embedding model lock not found/.test(error.message)) { + const recovery = /Embedding model lock not found/.test(error.message) + ? modelInstallHint(modelName, directory) + : `Remove or replace the invalid lock explicitly, then ${lowercaseFirst( + modelInstallHint(modelName, directory) + )}`; + throw new Error(`${error.message}. ${recovery}`, { cause: error }); + } + return installModelOnDemand(modelName, modelDir, installOptions, warn); } if (model.repository !== modelName) { @@ -57,19 +105,36 @@ async function resolveEmbeddingModel(configuration, options = {}) { try { await verifyModelDirectory(modelDir, model); } catch (error) { - throw new Error(`${error.message}. ${modelInstallHint(directory)}`, { - cause: error - }); + if (directory !== undefined) { + throw new Error(`${error.message}. ${modelInstallHint(modelName, directory)}`, { + cause: error + }); + } + return installModelOnDemand(modelName, modelDir, installOptions, warn); } return { model, modelDir }; } +async function installModelOnDemand(modelName, modelDir, options, warn) { + warn( + `Embedding model '${modelName}' is not available in '${modelDir}'. Downloading it now; application startup may be delayed. ${modelInstallHint(modelName)}` + ); + try { + return await installModel(modelName, options); + } catch (error) { + throw new Error( + `Failed to install embedding model '${modelName}': ${error.message}. ${modelInstallHint(modelName)}`, + { cause: error } + ); + } +} + function normalizeEmbeddingConfiguration(configuration) { if (configuration == null) { throw new Error('cds.env.requires.db.embedding.model must be a non-empty string'); } if (typeof configuration !== 'object' || Array.isArray(configuration)) { - throw new TypeError('embedding must be an object with model and directory'); + throw new TypeError('embedding must be an object with model and optional directory'); } const unsupported = Object.keys(configuration).filter( @@ -83,14 +148,18 @@ function normalizeEmbeddingConfiguration(configuration) { if (typeof configuration.model !== 'string' || !configuration.model.trim()) { throw new Error('cds.env.requires.db.embedding.model must be a non-empty string'); } - if (typeof configuration.directory !== 'string' || !configuration.directory.trim()) { - throw new Error('cds.env.requires.db.embedding.directory must be a non-empty string'); + if ( + configuration.directory !== undefined && + (typeof configuration.directory !== 'string' || !configuration.directory.trim()) + ) { + throw new Error('embedding.directory must be a non-empty string'); } return { model: configuration.model, directory: configuration.directory }; } -function modelInstallHint(directory) { - return `Run 'npx cds-ai model install --descriptor --directory ${directory}'.`; +function modelInstallHint(repository, directory) { + const directoryArgument = directory === undefined ? '' : ` --directory ${directory}`; + return `Run 'npx @cap-js/ai install-model ${repository}${directoryArgument}'.`; } function lowercaseFirst(value) { @@ -340,12 +409,14 @@ function embedding(text) { export { createSession, createEmbeddingRuntime, + createEmbeddingRuntimeFromModel, createFeeds, createTokenizerState, embedding, poolOutput, resolveEmbeddingModel, - tokenizeWithChunks + tokenizeWithChunks, + validateEmbeddingModel }; export default embedding; diff --git a/lib/vector_embedding/model-discovery.js b/lib/vector_embedding/model-discovery.js new file mode 100644 index 0000000..b2a716f --- /dev/null +++ b/lib/vector_embedding/model-discovery.js @@ -0,0 +1,390 @@ +import { createHash } from 'crypto'; + +import { assertSafeRepository, validateModelDescriptor } from './model-utils.js'; + +const HUGGING_FACE_ORIGIN = 'https://huggingface.co'; +const REQUIRED_ARTIFACTS = [ + { role: 'model', name: 'model.onnx', path: 'onnx/model.onnx' }, + { role: 'tokenizer', name: 'tokenizer.json', path: 'tokenizer.json' }, + { + role: 'tokenizerConfig', + name: 'tokenizer_config.json', + path: 'tokenizer_config.json' + }, + { role: 'auxiliary', name: 'config.json', path: 'config.json' } +]; +const MODULES_FILE = 'modules.json'; +const SENTENCE_CONFIG_FILE = 'sentence_bert_config.json'; +const SUPPORTED_MODULES = new Set([ + 'sentence_transformers.models.Transformer', + 'sentence_transformers.models.Pooling', + 'sentence_transformers.models.Normalize' +]); + +async function discoverModel(repository, options = {}) { + assertSafeRepository(repository); + const fetchImpl = + typeof options === 'function' ? options : (options.fetchImpl ?? globalThis.fetch); + if (typeof fetchImpl !== 'function') throw new TypeError('A fetch implementation is required'); + + const context = createContext( + fetchImpl, + typeof options === 'function' ? undefined : options.origin + ); + const modelInfo = await fetchModelInfo(context, repository); + const revision = immutableRevision(modelInfo, repository); + const siblings = siblingMap(modelInfo, repository); + + const tokenizer = await fetchJsonFile(context, repository, revision, 'tokenizer.json'); + const tokenizerConfig = await fetchJsonFile( + context, + repository, + revision, + 'tokenizer_config.json' + ); + const config = await fetchJsonFile(context, repository, revision, 'config.json'); + const dimensions = positiveInteger(config.value.hidden_size); + if (!dimensions) { + throw new Error(`Cannot determine embedding dimensions from '${repository}/config.json'`); + } + + const selected = REQUIRED_ARTIFACTS.map((artifact) => { + const sibling = siblings.get(artifact.path); + if (!sibling) { + throw new Error( + `Hugging Face model '${repository}' must contain the exact file '${artifact.path}'` + ); + } + return { ...artifact, sibling }; + }); + const externalData = [...siblings.values()] + .filter(({ rfilename }) => /^onnx\/model\.onnx_data(?:$|[._-])/u.test(rfilename)) + .map((sibling) => ({ + role: 'auxiliary', + name: sibling.rfilename.slice('onnx/'.length), + path: sibling.rfilename, + sibling + })); + if (usesExternalData(config.value) && externalData.length === 0) { + throw new Error( + `Hugging Face model '${repository}' declares external ONNX data but does not contain 'onnx/model.onnx_data'` + ); + } + selected.push(...externalData); + + const semantics = await discoverSentenceTransformerSemantics( + context, + repository, + modelInfo, + new Set() + ); + const maxLength = minimumPositiveInteger([ + tokenizer.value?.truncation?.max_length, + tokenizerConfig.value.max_length, + semantics.maxLength, + tokenizerConfig.value.model_max_length, + config.value.max_position_embeddings + ]); + if (!maxLength) { + throw new Error(`Cannot determine the maximum input length for '${repository}'`); + } + + const knownFiles = new Map([ + ['tokenizer.json', tokenizer], + ['tokenizer_config.json', tokenizerConfig], + ['config.json', config] + ]); + const files = await Promise.all( + selected.map(async ({ sibling, ...artifact }) => ({ + ...artifact, + ...(await discoverFileIntegrity( + context, + repository, + revision, + sibling, + knownFiles.get(artifact.path) + )) + })) + ); + + return validateModelDescriptor({ + repository, + revision, + dimensions, + maxLength, + files, + output: { + name: 'last_hidden_state', + pooling: semantics.pooling, + normalize: semantics.normalize + } + }); +} + +function createContext(fetchImpl, origin = HUGGING_FACE_ORIGIN) { + if (typeof origin !== 'string' || !origin.trim()) { + throw new TypeError('The Hugging Face origin must be a non-empty string'); + } + return { fetchImpl, origin: origin.replace(/\/$/, ''), jsonFiles: new Map() }; +} + +function usesExternalData(config) { + const value = config?.['transformers.js_config']?.use_external_data_format; + return value === true || value?.['model.onnx'] === 1 || value?.['model.onnx'] === true; +} + +async function fetchModelInfo(context, repository) { + const url = `${context.origin}/api/models/${repositoryPath(repository)}?blobs=true`; + const response = await checkedFetch(context, url); + const value = await readJsonResponse(response, url); + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`Invalid Hugging Face model information for '${repository}'`); + } + return value; +} + +function immutableRevision(modelInfo, repository) { + if (typeof modelInfo.sha !== 'string' || !/^[a-fA-F0-9]{40,64}$/.test(modelInfo.sha)) { + throw new Error(`Hugging Face did not return an immutable revision for '${repository}'`); + } + return modelInfo.sha.toLowerCase(); +} + +function siblingMap(modelInfo, repository) { + if (!Array.isArray(modelInfo.siblings)) { + throw new Error(`Hugging Face did not return a file list for '${repository}'`); + } + const siblings = new Map(); + for (const sibling of modelInfo.siblings) { + if (typeof sibling?.rfilename !== 'string') continue; + if (siblings.has(sibling.rfilename)) { + throw new Error(`Hugging Face returned duplicate file '${sibling.rfilename}'`); + } + siblings.set(sibling.rfilename, sibling); + } + return siblings; +} + +async function discoverSentenceTransformerSemantics(context, repository, modelInfo, visited) { + if (visited.has(repository)) { + throw new Error(`Circular Hugging Face base_model chain involving '${repository}'`); + } + visited.add(repository); + + const revision = immutableRevision(modelInfo, repository); + const siblings = siblingMap(modelInfo, repository); + if (siblings.has(MODULES_FILE)) { + return readSentenceTransformerSemantics(context, repository, revision, siblings); + } + + const baseModel = baseModelRepository(modelInfo.cardData?.base_model); + if (!baseModel) { + throw new Error( + `Cannot determine pooling and normalization for '${repository}': no Sentence Transformers modules or unambiguous base_model metadata` + ); + } + assertSafeRepository(baseModel); + const baseInfo = await fetchModelInfo(context, baseModel); + return discoverSentenceTransformerSemantics(context, baseModel, baseInfo, visited); +} + +async function readSentenceTransformerSemantics(context, repository, revision, siblings) { + const modules = (await fetchJsonFile(context, repository, revision, MODULES_FILE)).value; + if (!Array.isArray(modules)) { + throw new Error(`Invalid Sentence Transformers modules in '${repository}/${MODULES_FILE}'`); + } + + for (const module of modules) { + if (!module || typeof module.type !== 'string' || !SUPPORTED_MODULES.has(module.type)) { + throw new Error( + `Unsupported Sentence Transformers module '${module?.type ?? 'unknown'}' in '${repository}'` + ); + } + } + const expectedTypes = [ + 'sentence_transformers.models.Transformer', + 'sentence_transformers.models.Pooling' + ]; + if (modules.length === 3) expectedTypes.push('sentence_transformers.models.Normalize'); + if ( + modules.length < 2 || + modules.length > 3 || + modules.some((module, index) => module.type !== expectedTypes[index]) + ) { + throw new Error(`Cannot determine an unambiguous pooling pipeline for '${repository}'`); + } + + const poolingModule = modules[1]; + const poolingPath = moduleConfigPath(poolingModule, repository); + if (!siblings.has(poolingPath)) { + throw new Error(`Sentence Transformers pooling configuration '${poolingPath}' is missing`); + } + const poolingConfig = (await fetchJsonFile(context, repository, revision, poolingPath)).value; + const pooling = determinePooling(poolingConfig, repository); + + let maxLength; + const transformer = modules[0]; + const sentenceConfigPaths = [SENTENCE_CONFIG_FILE]; + if (transformer?.path) { + sentenceConfigPaths.unshift( + `${normalizedModulePath(transformer.path)}/${SENTENCE_CONFIG_FILE}` + ); + } + const sentenceConfigPath = sentenceConfigPaths.find((configPath) => siblings.has(configPath)); + if (sentenceConfigPath) { + const sentenceConfig = await fetchJsonFile(context, repository, revision, sentenceConfigPath); + maxLength = positiveInteger(sentenceConfig.value.max_seq_length); + } + + return { pooling, normalize: modules.length === 3, maxLength }; +} + +function moduleConfigPath(module, repository) { + const modulePath = normalizedModulePath(module.path); + if (!modulePath) { + throw new Error(`Sentence Transformers pooling module in '${repository}' has no path`); + } + return `${modulePath}/config.json`; +} + +function normalizedModulePath(value) { + if ( + typeof value !== 'string' || + !value || + value.includes('\\') || + value.startsWith('/') || + value.split('/').some((part) => !part || part === '.' || part === '..') + ) { + return undefined; + } + return value; +} + +function determinePooling(config, repository) { + if (!config || typeof config !== 'object' || Array.isArray(config)) { + throw new Error(`Invalid Sentence Transformers pooling configuration for '${repository}'`); + } + const enabled = [ + ['cls', config.pooling_mode_cls_token], + ['mean', config.pooling_mode_mean_tokens], + ['max', config.pooling_mode_max_tokens], + ['mean_sqrt_len', config.pooling_mode_mean_sqrt_len_tokens], + ['weightedmean', config.pooling_mode_weightedmean_tokens], + ['lasttoken', config.pooling_mode_lasttoken] + ].filter(([, value]) => value === true); + if (config.pooling_mode !== undefined) { + if (!['mean', 'cls'].includes(config.pooling_mode)) { + throw new Error(`Unsupported or ambiguous Sentence Transformers pooling for '${repository}'`); + } + if (enabled.length > 0 && (enabled.length !== 1 || enabled[0][0] !== config.pooling_mode)) { + throw new Error(`Unsupported or ambiguous Sentence Transformers pooling for '${repository}'`); + } + return config.pooling_mode; + } + if (enabled.length !== 1 || !['mean', 'cls'].includes(enabled[0][0])) { + throw new Error(`Unsupported or ambiguous Sentence Transformers pooling for '${repository}'`); + } + return enabled[0][0]; +} + +function baseModelRepository(value) { + if (typeof value === 'string') return value; + if (Array.isArray(value) && value.length === 1 && typeof value[0] === 'string') return value[0]; + if (value && typeof value === 'object' && !Array.isArray(value) && typeof value.id === 'string') { + return value.id; + } + return undefined; +} + +async function discoverFileIntegrity(context, repository, revision, sibling, knownFile) { + const metadataChecksum = lfsChecksum(sibling); + const metadataSize = positiveInteger(sibling.size) ?? positiveInteger(sibling.lfs?.size); + if (metadataChecksum && metadataSize) { + return { size: metadataSize, sha256: metadataChecksum }; + } + + const file = knownFile ?? (await fetchFile(context, repository, revision, sibling.rfilename)); + return { + size: file.bytes.byteLength, + sha256: createHash('sha256').update(file.bytes).digest('hex') + }; +} + +function lfsChecksum(sibling) { + const candidate = sibling.lfs?.sha256 ?? sibling.lfs?.oid; + if (typeof candidate !== 'string') return undefined; + const checksum = candidate.replace(/^sha256:/, '').toLowerCase(); + return /^[a-f0-9]{64}$/.test(checksum) ? checksum : undefined; +} + +async function fetchJsonFile(context, repository, revision, remotePath) { + const key = `${repository}@${revision}/${remotePath}`; + let file = context.jsonFiles.get(key); + if (!file) { + file = fetchFile(context, repository, revision, remotePath).then(({ bytes, url }) => { + try { + return { bytes, value: JSON.parse(bytes.toString('utf8')) }; + } catch (error) { + throw new Error(`Invalid JSON returned from ${url}: ${error.message}`, { cause: error }); + } + }); + context.jsonFiles.set(key, file); + } + return file; +} + +async function fetchFile(context, repository, revision, remotePath) { + const url = `${context.origin}/${repositoryPath(repository)}/resolve/${revision}/${remotePath + .split('/') + .map(encodeURIComponent) + .join('/')}`; + const response = await checkedFetch(context, url); + return { bytes: await readBytes(response), url }; +} + +async function checkedFetch(context, url) { + let response; + try { + response = await context.fetchImpl(url); + } catch (error) { + throw new Error(`Cannot fetch ${url}: ${error.message}`, { cause: error }); + } + if (!response || response.ok !== true) { + throw new Error(`Cannot fetch ${url}: HTTP ${response?.status ?? 'unknown'}`); + } + return response; +} + +async function readJsonResponse(response, url) { + try { + if (typeof response.json === 'function') return await response.json(); + return JSON.parse((await readBytes(response)).toString('utf8')); + } catch (error) { + throw new Error(`Invalid JSON returned from ${url}: ${error.message}`, { cause: error }); + } +} + +async function readBytes(response) { + if (typeof response.arrayBuffer === 'function') { + return Buffer.from(await response.arrayBuffer()); + } + if (typeof response.text === 'function') return Buffer.from(await response.text()); + throw new Error('Fetch response does not expose arrayBuffer() or text()'); +} + +function positiveInteger(value) { + return Number.isSafeInteger(value) && value > 0 ? value : undefined; +} + +function minimumPositiveInteger(values) { + const candidates = values.map(positiveInteger).filter((value) => value !== undefined); + return candidates.length > 0 ? Math.min(...candidates) : undefined; +} + +function repositoryPath(repository) { + return repository.split('/').map(encodeURIComponent).join('/'); +} + +const discoverModelDescriptor = discoverModel; + +export { discoverModel, discoverModelDescriptor }; diff --git a/lib/vector_embedding/model-install.js b/lib/vector_embedding/model-install.js new file mode 100644 index 0000000..13f2ed3 --- /dev/null +++ b/lib/vector_embedding/model-install.js @@ -0,0 +1,64 @@ +import { setTimeout as delay } from 'node:timers/promises'; +import { discoverModel } from './model-discovery.js'; +import { + MODEL_PROVISIONING_IN_PROGRESS, + assertSafeRepository, + getModelDirectory, + getModelRoot, + provisionModel, + readModelLock +} from './model-utils.js'; + +const MODEL_PROVISION_TIMEOUT_MS = 15 * 60 * 1000; +const MODEL_PROVISION_RETRY_MS = 250; + +async function installModel(repository, options = {}) { + assertSafeRepository(repository); + const modelRoot = getModelRoot(options.directory, options.root, options.home); + const modelDir = getModelDirectory(modelRoot, repository); + const discover = options.discover ?? discoverModel; + + let model; + try { + model = await readModelLock(modelDir); + assertRepository(model, repository, modelDir); + } catch (error) { + if (!/Embedding model lock not found/.test(error.message)) throw error; + model = await discover(repository, { fetchImpl: options.fetchImpl }); + assertRepository(model, repository, modelDir); + } + + const deadline = Date.now() + (options.timeoutMs ?? MODEL_PROVISION_TIMEOUT_MS); + const retryMs = options.retryMs ?? MODEL_PROVISION_RETRY_MS; + while (true) { + try { + // eslint-disable-next-line no-await-in-loop + await provisionModel(model, { + directory: modelDir, + fetchImpl: options.fetchImpl, + validate: options.validate + }); + return { model, modelDir, modelRoot }; + } catch (error) { + if (error.code !== MODEL_PROVISIONING_IN_PROGRESS) throw error; + const remaining = deadline - Date.now(); + if (remaining <= 0) { + throw new Error(`Timed out waiting for embedding model provisioning in ${modelDir}`, { + cause: error + }); + } + // eslint-disable-next-line no-await-in-loop + await delay(Math.min(retryMs, remaining)); + } + } +} + +function assertRepository(model, repository, modelDir) { + if (model.repository !== repository) { + throw new Error( + `Embedding model directory ${modelDir} contains ${model.repository}, not ${repository}. Choose another directory or remove it explicitly before installing the configured model.` + ); + } +} + +export { installModel }; diff --git a/lib/vector_embedding/model-utils.js b/lib/vector_embedding/model-utils.js index e87c135..5fe5728 100644 --- a/lib/vector_embedding/model-utils.js +++ b/lib/vector_embedding/model-utils.js @@ -106,6 +106,18 @@ function assertSafeRepository(repository) { } } +function getModelRoot(directory, root = process.cwd(), home = os.homedir()) { + if (directory === undefined) return path.join(root, '.cds', 'models'); + if (directory === '~') return home; + if (/^~[\\/]/.test(directory)) return path.join(home, directory.slice(2)); + return path.resolve(root, directory); +} + +function getModelDirectory(root, repository) { + assertSafeRepository(repository); + return path.join(root, ...repository.split('/')); +} + function assertSafeRelativePath(value, field) { const parts = typeof value === 'string' ? value.split('/') : []; if ( @@ -498,6 +510,7 @@ async function provisionModel(model, options = {}) { await verifyModelDirectory(directory, model); if (!lockedModel) await writeModelLock(directory, model); await makeModelDirectoryReadable(directory, model); + await options.validate?.(directory, model); return directory; } catch (error) { if (/symbolic link/.test(error.message)) throw error; @@ -516,6 +529,7 @@ async function provisionModel(model, options = {}) { await downloadModelIfNeeded(stagingDirectory, model, options); await verifyModelDirectory(stagingDirectory, model); await writeModelLock(stagingDirectory, model); + await options.validate?.(stagingDirectory, model); await publishModelDirectory(stagingDirectory, directory); published = true; } finally { @@ -707,18 +721,23 @@ async function loadModelAndTokenizer(modelDir, model) { fs.readFile(tokenizerPath, 'utf8').then(JSON.parse), fs.readFile(tokenizerConfigPath, 'utf8').then(JSON.parse) ]); + const tokenizer = new Tokenizer(tokenizerJson, tokenizerConfig); return { session: await InferenceSession.create(modelPath), - tokenizer: new Tokenizer(tokenizerJson, tokenizerConfig) + tokenizer }; } export { MODEL_LOCK_FILE, + MODEL_PROVISIONING_IN_PROGRESS, + assertSafeRepository, downloadFile, downloadModelIfNeeded, fileForRole, + getModelDirectory, + getModelRoot, isValidFile, loadModelAndTokenizer, modelDescriptorDigest, diff --git a/package.json b/package.json index f38cc49..701bb9c 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ }, "scripts": { "lint": "npx -y eslint@10 .", - "test:model:provision": "node bin/cds-ai.js model install --descriptor tests/fixtures/minilm.embedding-model.json --directory tests/.models/minilm", + "test:model:provision": "node bin/cds-ai.js install-model Xenova/all-MiniLM-L6-v2", "pretest": "npm run test:model:provision", "pretest:hybrid": "npm run test:model:provision", "test": "node --test tests/*.test.js", diff --git a/tests/fixtures/minilm.embedding-model.json b/tests/fixtures/minilm.embedding-model.json deleted file mode 100644 index 2185c0b..0000000 --- a/tests/fixtures/minilm.embedding-model.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "repository": "Xenova/all-MiniLM-L6-v2", - "revision": "751bff37182d3f1213fa05d7196b954e230abad9", - "dimensions": 384, - "maxLength": 128, - "files": [ - { - "role": "model", - "name": "model.onnx", - "path": "onnx/model.onnx", - "size": 90387606, - "sha256": "759c3cd2b7fe7e93933ad23c4c9181b7396442a2ed746ec7c1d46192c469c46e" - }, - { - "role": "tokenizer", - "name": "tokenizer.json", - "path": "tokenizer.json", - "size": 711661, - "sha256": "da0e79933b9ed51798a3ae27893d3c5fa4a201126cef75586296df9b4d2c62a0" - }, - { - "role": "tokenizerConfig", - "name": "tokenizer_config.json", - "path": "tokenizer_config.json", - "size": 366, - "sha256": "9261e7d79b44c8195c1cada2b453e55b00aeb81e907a6664974b4d7776172ab3" - } - ], - "output": { - "name": "last_hidden_state", - "pooling": "mean", - "normalize": true - } -} diff --git a/tests/model-discovery.test.js b/tests/model-discovery.test.js new file mode 100644 index 0000000..e6020e2 --- /dev/null +++ b/tests/model-discovery.test.js @@ -0,0 +1,326 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { describe, test } from 'node:test'; + +import { discoverModel } from '../lib/vector_embedding/model-discovery.js'; + +const REVISION = '1'.repeat(40); +const BASE_REVISION = '2'.repeat(40); +const REPOSITORY = 'example/embedding-model'; +const BASE_REPOSITORY = 'sentence-transformers/base-model'; + +describe('Hugging Face model discovery', () => { + test('creates a validated descriptor from exact artifacts and Sentence Transformers metadata', async () => { + const routes = modelRoutes({ + tokenizer: { truncation: { max_length: 96 } }, + tokenizerConfig: { model_max_length: 128 }, + config: { hidden_size: 384, max_position_embeddings: 512 }, + sentenceConfig: { max_seq_length: 256 }, + normalize: true + }); + + const descriptor = await discoverModel(REPOSITORY, { fetchImpl: createFetch(routes) }); + + assert.equal(descriptor.repository, REPOSITORY); + assert.equal(descriptor.revision, REVISION); + assert.equal(descriptor.dimensions, 384); + assert.equal(descriptor.maxLength, 96); + assert.deepEqual(descriptor.output, { + name: 'last_hidden_state', + pooling: 'mean', + normalize: true + }); + assert.deepEqual( + descriptor.files.map(({ role, name, path }) => ({ role, name, path })), + [ + { role: 'model', name: 'model.onnx', path: 'onnx/model.onnx' }, + { role: 'tokenizer', name: 'tokenizer.json', path: 'tokenizer.json' }, + { + role: 'tokenizerConfig', + name: 'tokenizer_config.json', + path: 'tokenizer_config.json' + }, + { role: 'auxiliary', name: 'config.json', path: 'config.json' } + ] + ); + const tokenizerFile = descriptor.files.find(({ role }) => role === 'tokenizer'); + assert.equal( + tokenizerFile.sha256, + digest(routes[fileUrl(REPOSITORY, REVISION, 'tokenizer.json')]) + ); + assert.equal( + tokenizerFile.size, + routes[fileUrl(REPOSITORY, REVISION, 'tokenizer.json')].length + ); + }); + + test('follows a pinned base_model for semantics and Sentence Transformers max length', async () => { + const routes = modelRoutes({ + tokenizer: { truncation: null }, + tokenizerConfig: { model_max_length: 128 }, + config: { hidden_size: 768, max_position_embeddings: 512 }, + modules: false, + baseModel: BASE_REPOSITORY + }); + addBaseModelRoutes(routes, { + sentenceConfig: { max_seq_length: 64 }, + pooling: 'cls', + normalize: false + }); + + const requested = []; + const descriptor = await discoverModel(REPOSITORY, { + fetchImpl: createFetch(routes, requested) + }); + + assert.equal(descriptor.maxLength, 64); + assert.equal(descriptor.output.pooling, 'cls'); + assert.equal(descriptor.output.normalize, false); + assert.ok( + requested.includes(apiUrl(BASE_REPOSITORY)), + 'the base repository is resolved through the model API' + ); + assert.ok( + requested.some((url) => url.includes(`/resolve/${BASE_REVISION}/modules.json`)), + 'base-model metadata is read from its immutable revision' + ); + }); + + test('falls back from tokenizer configuration to model configuration for max length', async () => { + const tokenizerRoutes = modelRoutes({ + tokenizer: { truncation: null }, + tokenizerConfig: { model_max_length: 192 }, + config: { hidden_size: 32, max_position_embeddings: 256 }, + sentenceConfig: undefined + }); + assert.equal( + (await discoverModel(REPOSITORY, { fetchImpl: createFetch(tokenizerRoutes) })).maxLength, + 192 + ); + + const configRoutes = modelRoutes({ + tokenizer: { truncation: null }, + tokenizerConfig: { model_max_length: 1e30 }, + config: { hidden_size: 32, max_position_embeddings: 256 }, + sentenceConfig: undefined + }); + assert.equal( + (await discoverModel(REPOSITORY, { fetchImpl: createFetch(configRoutes) })).maxLength, + 256 + ); + }); + + test('uses the lowest declared tokenizer and model input limit', async () => { + const routes = modelRoutes({ + tokenizer: { truncation: null }, + tokenizerConfig: { max_length: 128, model_max_length: 512 }, + config: { hidden_size: 32, max_position_embeddings: 256 }, + sentenceConfig: { max_seq_length: 384 } + }); + + assert.equal( + (await discoverModel(REPOSITORY, { fetchImpl: createFetch(routes) })).maxLength, + 128 + ); + }); + + test('includes external ONNX data files', async () => { + const routes = modelRoutes({ externalData: true }); + const descriptor = await discoverModel(REPOSITORY, { fetchImpl: createFetch(routes) }); + + assert.deepEqual( + descriptor.files.find(({ path }) => path === 'onnx/model.onnx_data'), + { + role: 'auxiliary', + name: 'model.onnx_data', + path: 'onnx/model.onnx_data', + size: routes[fileUrl(REPOSITORY, REVISION, 'onnx/model.onnx_data')].length, + sha256: digest(routes[fileUrl(REPOSITORY, REVISION, 'onnx/model.onnx_data')]) + } + ); + }); + + test('rejects missing exact artifacts and ambiguous runtime semantics', async () => { + const missing = modelRoutes({}); + const info = JSON.parse(missing[apiUrl(REPOSITORY)].toString()); + info.siblings = info.siblings.filter(({ rfilename }) => rfilename !== 'onnx/model.onnx'); + missing[apiUrl(REPOSITORY)] = json(info); + await assert.rejects( + discoverModel(REPOSITORY, { fetchImpl: createFetch(missing) }), + /exact file 'onnx\/model\.onnx'/ + ); + + const ambiguous = modelRoutes({ pooling: ['mean', 'cls'] }); + await assert.rejects( + discoverModel(REPOSITORY, { fetchImpl: createFetch(ambiguous) }), + /Unsupported or ambiguous Sentence Transformers pooling/ + ); + + const invalidOrder = modelRoutes({ moduleOrder: ['Pooling', 'Transformer'] }); + await assert.rejects( + discoverModel(REPOSITORY, { fetchImpl: createFetch(invalidOrder) }), + /unambiguous pooling pipeline/ + ); + + const conflicting = modelRoutes({ pooling: 'mean', poolingMode: 'cls' }); + await assert.rejects( + discoverModel(REPOSITORY, { fetchImpl: createFetch(conflicting) }), + /Unsupported or ambiguous Sentence Transformers pooling/ + ); + }); + + test('downloads an artifact to derive integrity when Hugging Face has no LFS metadata', async () => { + const routes = modelRoutes({ modelMetadata: false }); + const descriptor = await discoverModel(REPOSITORY, createFetch(routes)); + const model = descriptor.files.find(({ role }) => role === 'model'); + const contents = routes[fileUrl(REPOSITORY, REVISION, 'onnx/model.onnx')]; + assert.equal(model.size, contents.length); + assert.equal(model.sha256, digest(contents)); + }); +}); + +function modelRoutes(options = {}) { + const tokenizer = json(options.tokenizer ?? { truncation: { max_length: 96 } }); + const tokenizerConfig = json(options.tokenizerConfig ?? { model_max_length: 128 }); + const configValue = options.config ?? { hidden_size: 384, max_position_embeddings: 512 }; + if (options.externalData) { + configValue['transformers.js_config'] = { + use_external_data_format: { 'model.onnx': 1 } + }; + } + const config = json(configValue); + const model = Buffer.from('fake onnx model'); + const moduleTypes = options.moduleOrder ?? [ + 'Transformer', + 'Pooling', + ...(options.normalize === false ? [] : ['Normalize']) + ]; + const modules = json( + moduleTypes.map((type, index) => ({ + idx: index, + name: String(index), + path: type === 'Pooling' ? '1_Pooling' : '', + type: `sentence_transformers.models.${type}` + })) + ); + const pooling = json(poolingConfig(options.pooling ?? 'mean', options.poolingMode)); + const files = { + 'onnx/model.onnx': model, + 'tokenizer.json': tokenizer, + 'tokenizer_config.json': tokenizerConfig, + 'config.json': config + }; + if (options.externalData) files['onnx/model.onnx_data'] = Buffer.from('external weights'); + if (options.modules !== false) { + files['modules.json'] = modules; + files['1_Pooling/config.json'] = pooling; + if (options.sentenceConfig !== undefined) { + files['sentence_bert_config.json'] = json(options.sentenceConfig); + } else if (!Object.hasOwn(options, 'sentenceConfig')) { + files['sentence_bert_config.json'] = json({ max_seq_length: 256 }); + } + } + const siblings = Object.entries(files).map(([rfilename, contents]) => ({ + rfilename, + ...(rfilename === 'onnx/model.onnx' && options.modelMetadata !== false + ? { size: contents.length, lfs: { size: contents.length, sha256: digest(contents) } } + : {}) + })); + return { + [apiUrl(REPOSITORY)]: json({ + sha: REVISION, + siblings, + ...(options.baseModel ? { cardData: { base_model: options.baseModel } } : {}) + }), + ...Object.fromEntries( + Object.entries(files).map(([name, contents]) => [ + fileUrl(REPOSITORY, REVISION, name), + contents + ]) + ) + }; +} + +function addBaseModelRoutes(routes, options = {}) { + const modules = json([ + { + idx: 0, + path: '', + type: 'sentence_transformers.models.Transformer' + }, + { + idx: 1, + path: '1_Pooling', + type: 'sentence_transformers.models.Pooling' + }, + ...(options.normalize + ? [{ idx: 2, path: '2_Normalize', type: 'sentence_transformers.models.Normalize' }] + : []) + ]); + const pooling = json(poolingConfig(options.pooling ?? 'mean')); + const sentenceConfig = json(options.sentenceConfig ?? { max_seq_length: 128 }); + const files = { + 'modules.json': modules, + '1_Pooling/config.json': pooling, + 'sentence_bert_config.json': sentenceConfig + }; + routes[apiUrl(BASE_REPOSITORY)] = json({ + sha: BASE_REVISION, + siblings: Object.keys(files).map((rfilename) => ({ rfilename })) + }); + for (const [name, contents] of Object.entries(files)) { + routes[fileUrl(BASE_REPOSITORY, BASE_REVISION, name)] = contents; + } +} + +function poolingConfig(pooling, poolingMode) { + const enabled = Array.isArray(pooling) ? pooling : [pooling]; + return { + ...(poolingMode ? { pooling_mode: poolingMode } : {}), + pooling_mode_cls_token: enabled.includes('cls'), + pooling_mode_mean_tokens: enabled.includes('mean'), + pooling_mode_max_tokens: enabled.includes('max'), + pooling_mode_mean_sqrt_len_tokens: false, + pooling_mode_weightedmean_tokens: false, + pooling_mode_lasttoken: false + }; +} + +function createFetch(routes, requested = []) { + return async (input) => { + const url = String(input); + requested.push(url); + const contents = routes[url]; + if (!contents) return response(Buffer.alloc(0), 404); + return response(contents, 200); + }; +} + +function response(contents, status) { + return { + ok: status >= 200 && status < 300, + status, + async json() { + return JSON.parse(contents.toString()); + }, + async arrayBuffer() { + return contents; + } + }; +} + +function apiUrl(repository) { + return `https://huggingface.co/api/models/${repository}?blobs=true`; +} + +function fileUrl(repository, revision, name) { + return `https://huggingface.co/${repository}/resolve/${revision}/${name}`; +} + +function json(value) { + return Buffer.from(JSON.stringify(value)); +} + +function digest(value) { + return createHash('sha256').update(value).digest('hex'); +} diff --git a/tests/model-provisioning.test.js b/tests/model-provisioning.test.js index f5db013..01c7d0d 100644 --- a/tests/model-provisioning.test.js +++ b/tests/model-provisioning.test.js @@ -8,6 +8,8 @@ import { runModelCommand } from '../lib/vector_embedding/cli.js'; import { resolveEmbeddingModel } from '../lib/vector_embedding/embedding.js'; import { MODEL_LOCK_FILE, + getModelDirectory, + getModelRoot, provisionModel, readModelLock, verifyModelDirectory @@ -36,19 +38,15 @@ describe('runtime model configuration', () => { ); await assert.rejects( resolveEmbeddingModel(model.repository), - /embedding must be an object with model and directory/ + /embedding must be an object with model and optional directory/ ); await assert.rejects( resolveEmbeddingModel({ ...model }), /Only model and directory are supported/ ); - await assert.rejects( - resolveEmbeddingModel({ model: model.repository }), - /cds\.env\.requires\.db\.embedding\.directory must be a non-empty string/ - ); await assert.rejects( resolveEmbeddingModel({ model: model.repository, directory: '' }), - /cds\.env\.requires\.db\.embedding\.directory must be a non-empty string/ + /embedding\.directory must be a non-empty string/ ); }); }); @@ -306,6 +304,29 @@ describe('explicit model provisioning', () => { await assert.rejects(fs.access(path.join(directory, MODEL_LOCK_FILE))); }); + test('validates the runtime before publishing a newly installed model', async () => { + const parent = await createTemporaryDirectory(); + const directory = path.join(parent, 'model'); + const content = Buffer.from('runtime validation fixture'); + const model = fixtureModel(content); + let stagedDirectory; + + await assert.rejects( + provisionModel(model, { + directory, + fetchImpl: createFetch(content), + validate(candidate) { + stagedDirectory = candidate; + throw new Error('incompatible ONNX runtime'); + } + }), + /incompatible ONNX runtime/ + ); + + assert.notEqual(stagedDirectory, directory); + await assert.rejects(fs.access(directory)); + }); + test('fails verification instead of downloading missing runtime files', async () => { const directory = await createTemporaryDirectory(); const model = fixtureModel(Buffer.from('fixture')); @@ -317,6 +338,91 @@ describe('explicit model provisioning', () => { assert.deepEqual(await fs.readdir(directory), []); }); + test('downloads a missing model into the project-local default directory and reuses it', async () => { + const root = await createTemporaryDirectory(); + const content = Buffer.from('lazy download fixture'); + const model = fixtureModel(content); + const requestedUrls = []; + const warnings = []; + let discoveries = 0; + const options = { + root, + fetchImpl: createFetch(content, requestedUrls), + discover(name) { + discoveries++; + assert.equal(name, model.repository); + return model; + }, + validate: async () => {}, + warn: (message) => warnings.push(message) + }; + + const first = await resolveEmbeddingModel({ model: model.repository }, options); + const expectedDirectory = getModelDirectory(getModelRoot(undefined, root), model.repository); + + assert.equal(first.model, model); + assert.equal(first.modelDir, expectedDirectory); + assert.equal(discoveries, 1); + assert.equal(warnings.length, 1); + assert.match(warnings[0], /Downloading it now; application startup may be delayed/); + assert.equal(requestedUrls.length, model.files.length); + assert.deepEqual(await readModelLock(expectedDirectory), model); + + const second = await resolveEmbeddingModel({ model: model.repository }, options); + assert.equal(second.modelDir, expectedDirectory); + assert.equal(discoveries, 1); + assert.equal(warnings.length, 1); + assert.equal(requestedUrls.length, model.files.length); + }); + + test('waits for concurrent ad-hoc provisioning and reuses the completed download', async () => { + const root = await createTemporaryDirectory(); + const content = Buffer.from('concurrent lazy download fixture'); + const model = fixtureModel(content); + const requestedUrls = []; + const warnings = []; + let releaseDownload; + let signalDownloadStarted; + let firstRequest = true; + const downloadStarted = new Promise((resolve) => { + signalDownloadStarted = resolve; + }); + const waitForRelease = new Promise((resolve) => { + releaseDownload = resolve; + }); + const fetchImpl = async (url) => { + requestedUrls.push(url); + if (firstRequest) { + firstRequest = false; + signalDownloadStarted(); + await waitForRelease; + } + return new Response(content, { + headers: { 'content-length': String(content.length) } + }); + }; + const options = { + root, + fetchImpl, + discover: () => model, + validate: async () => {}, + warn: (message) => warnings.push(message), + provisionRetryMs: 5, + provisionTimeoutMs: 1000 + }; + + const first = resolveEmbeddingModel({ model: model.repository }, options); + await downloadStarted; + const second = resolveEmbeddingModel({ model: model.repository }, options); + await new Promise((resolve) => setTimeout(resolve, 20)); + releaseDownload(); + + const resolved = await Promise.all([first, second]); + assert.equal(resolved[0].modelDir, resolved[1].modelDir); + assert.equal(warnings.length, 2); + assert.equal(requestedUrls.length, model.files.length); + }); + test('keeps explicitly configured directories offline', async () => { const root = await createTemporaryDirectory(); let fetched = false; @@ -334,100 +440,112 @@ describe('explicit model provisioning', () => { } } ), - /cds-ai model install --descriptor --directory \.\/models\/minilm/ + /@cap-js\/ai install-model example\/model --directory \.\/models/ ); assert.equal(fetched, false); }); test('resolves relative directories from cds.root and preserves absolute directories', async () => { const root = await createTemporaryDirectory(); - const directory = path.join(root, 'models', 'custom'); const content = Buffer.from('directory resolution fixture'); const model = fixtureModel(content); - await provisionModel(model, { directory, fetchImpl: createFetch(content) }); + const modelRoot = path.join(root, 'models'); + const modelDir = getModelDirectory(modelRoot, model.repository); + await provisionModel(model, { directory: modelDir, fetchImpl: createFetch(content) }); const relative = await resolveEmbeddingModel( - { model: model.repository, directory: './models/custom' }, + { model: model.repository, directory: './models' }, { root } ); const absolute = await resolveEmbeddingModel( - { model: model.repository, directory }, + { model: model.repository, directory: modelRoot }, { root: await createTemporaryDirectory() } ); - assert.equal(relative.modelDir, directory); - assert.equal(absolute.modelDir, directory); + assert.equal(relative.modelDir, modelDir); + assert.equal(absolute.modelDir, modelDir); }); test('rejects a configured model name that does not match the provisioned lock', async () => { - const directory = await createTemporaryDirectory(); + const modelRoot = await createTemporaryDirectory(); const content = Buffer.from('repository mismatch fixture'); const model = fixtureModel(content); - await provisionModel(model, { directory, fetchImpl: createFetch(content) }); + const modelDir = getModelDirectory(modelRoot, 'example/other-model'); + await provisionModel(model, { directory: modelDir, fetchImpl: createFetch(content) }); await assert.rejects( - resolveEmbeddingModel({ model: 'example/other-model', directory }), + resolveEmbeddingModel({ model: 'example/other-model', directory: modelRoot }), /contains example\/model, not example\/other-model/ ); }); - test('gives models a descriptor-based provisioning command', async () => { + test('gives models a model-name provisioning command', async () => { const root = await createTemporaryDirectory(); await assert.rejects( resolveEmbeddingModel({ model: 'example/custom', directory: './models/custom' }, { root }), - /model install --descriptor --directory \.\/models\/custom/ + /@cap-js\/ai install-model example\/custom --directory \.\/models\/custom/ ); }); test('requires explicit lock recovery before reinstalling', async () => { - const directory = await createTemporaryDirectory(); - await fs.writeFile(path.join(directory, MODEL_LOCK_FILE), '{}'); + const modelRoot = await createTemporaryDirectory(); + const modelDir = getModelDirectory(modelRoot, 'example/model'); + await fs.mkdir(modelDir, { recursive: true }); + await fs.writeFile(path.join(modelDir, MODEL_LOCK_FILE), '{}'); await assert.rejects( - resolveEmbeddingModel({ model: 'example/model', directory }), - /Remove or replace the invalid lock explicitly, then run 'npx cds-ai model install/ + resolveEmbeddingModel({ model: 'example/model', directory: modelRoot }), + /Remove or replace the invalid lock explicitly, then run 'npx @cap-js\/ai install-model/ ); }); - test('installs a descriptor through the command API', async () => { + test('installs a model by name through the command API', async () => { const root = await createTemporaryDirectory(); - const directory = path.join(root, 'models', 'custom'); - const descriptor = path.join(root, 'embedding-model.json'); + const modelRoot = path.join(root, 'models'); const content = Buffer.from('command fixture'); const model = fixtureModel(content); const output = []; - await fs.writeFile(descriptor, JSON.stringify(model)); - await runModelCommand( - ['model', 'install', '--descriptor', descriptor, '--directory', directory], - { - cwd: root, - fetchImpl: createFetch(content), - stdout: { write: (value) => output.push(value) } - } - ); + await runModelCommand(['install-model', model.repository, '--directory', modelRoot], { + cwd: root, + discover: () => model, + fetchImpl: createFetch(content), + validate: async () => {}, + stdout: { write: (value) => output.push(value) } + }); - assert.deepEqual(await readModelLock(directory), model); - assert.match(output.join(''), /Provisioned example\/model/); + const modelDir = getModelDirectory(modelRoot, model.repository); + assert.deepEqual(await readModelLock(modelDir), model); + assert.match(output.join(''), /Installed example\/model/); + assert.match(output.join(''), new RegExp(escapeRegExp(modelDir))); }); - test('requires a descriptor and directory when provisioning', async () => { - await assert.rejects( - provisionModel(fixtureModel(Buffer.from('missing directory fixture'))), - /non-empty provisioning directory is required/ - ); - await assert.rejects( - runModelCommand(['model', 'install', '--descriptor', './embedding-model.json']), - /--directory is required/ - ); + test('installs into the project-local model cache when no directory is provided', async () => { + const root = await createTemporaryDirectory(); + const content = Buffer.from('default command fixture'); + const model = fixtureModel(content); + + await runModelCommand(['install-model', model.repository], { + cwd: root, + discover: () => model, + fetchImpl: createFetch(content), + validate: async () => {}, + stdout: { write() {} } + }); + + const modelDir = path.join(root, '.cds', 'models', 'example', 'model'); + assert.deepEqual(await readModelLock(modelDir), model); + }); + + test('requires a model name and accepts an optional cache root', async () => { await assert.rejects( - runModelCommand(['model', 'install', '--directory', './models/custom']), - /--descriptor is required/ + runModelCommand(['install-model', '--directory', './models/custom']), + /Specify a model name/ ); await assert.rejects( - runModelCommand(['model', 'install', 'example/model', '--directory', './models/custom']), - /Unexpected argument 'example\/model'/ + runModelCommand(['install-model', 'example/model', 'example/other']), + /Unexpected argument 'example\/other'/ ); }); }); diff --git a/tests/vector-unit.test.js b/tests/vector-unit.test.js index a59a59d..dc5bf45 100644 --- a/tests/vector-unit.test.js +++ b/tests/vector-unit.test.js @@ -4,6 +4,7 @@ import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { afterEach, describe, test } from 'node:test'; +import { InferenceSession } from '../lib/vector_embedding/InferenceSession.js'; import { createFeeds, createTokenizerState, @@ -13,11 +14,27 @@ import { import { downloadFile, downloadModelIfNeeded, + getModelDirectory, + getModelRoot, validateModelDescriptor } from '../lib/vector_embedding/model-utils.js'; const temporaryDirectories = []; +test('disposes inference sessions at most once', async () => { + let disposals = 0; + const session = new InferenceSession({ + dispose() { + disposals++; + } + }); + + await session.dispose(); + await session.dispose(); + + assert.equal(disposals, 1); +}); + afterEach(async () => { await Promise.all( temporaryDirectories @@ -165,6 +182,28 @@ describe('model compatibility', () => { }); }); +describe('model directories', () => { + test('uses a project-local default and appends the model repository', () => { + const project = path.join(path.sep, 'project'); + const root = getModelRoot(undefined, project); + + assert.equal(root, path.join(project, '.cds', 'models')); + assert.equal(getModelDirectory(root, 'foo/bar'), path.join(root, 'foo', 'bar')); + }); + + test('resolves relative, absolute, and home-relative roots', () => { + const project = path.join(path.sep, 'project'); + const home = path.join(path.sep, 'home', 'user'); + + assert.equal(getModelRoot('./models', project, home), path.join(project, 'models')); + assert.equal( + getModelRoot(path.join(path.sep, 'shared', 'models'), project, home), + path.join(path.sep, 'shared', 'models') + ); + assert.equal(getModelRoot('~/.cds/models', project, home), path.join(home, '.cds', 'models')); + }); +}); + describe('model download', () => { test('uses a pinned revision and atomically caches verified files', async () => { const directory = await createTemporaryDirectory(); diff --git a/tests/vector.test.js b/tests/vector.test.js index a0cd4b8..89a92b5 100644 --- a/tests/vector.test.js +++ b/tests/vector.test.js @@ -1,22 +1,14 @@ import { after, before, describe, test } from 'node:test'; import assert from 'node:assert'; -import fs from 'node:fs/promises'; -import { fileURLToPath } from 'node:url'; import cds from '@sap/cds'; import { initializeEmbedding, vector_embedding } from '../lib/vector_embedding/index.js'; -const MINILM_MODEL = JSON.parse( - await fs.readFile(new URL('./fixtures/minilm.embedding-model.json', import.meta.url), 'utf8') -); -const MINILM_DIRECTORY = fileURLToPath(new URL('./.models/minilm', import.meta.url)); +const MINILM_MODEL = 'Xenova/all-MiniLM-L6-v2'; let embeddingModule; before(async () => { - embeddingModule = await initializeEmbedding({ - model: MINILM_MODEL.repository, - directory: MINILM_DIRECTORY - }); + embeddingModule = await initializeEmbedding({ model: MINILM_MODEL }); }); describe('Vector embedding function (standalone)', () => { @@ -152,24 +144,10 @@ describe('ai-sqlite integration', () => { ); }); - test('requires an explicitly configured embedding directory during startup', async () => { - await assert.rejects( - cds.connect.to('missing-embedding-directory-db', { - kind: 'ai-sqlite', - embedding: { model: MINILM_MODEL.repository }, - credentials: { url: ':memory:' } - }), - /cds\.env\.requires\.db\.embedding\.directory must be a non-empty string/ - ); - }); - before(async () => { db = await cds.connect.to('vector-db', { kind: 'ai-sqlite', - embedding: { - model: MINILM_MODEL.repository, - directory: MINILM_DIRECTORY - }, + embedding: { model: MINILM_MODEL }, credentials: { url: ':memory:' } }); }); @@ -199,7 +177,7 @@ describe('ai-sqlite integration', () => { await assert.rejects( cds.connect.to('invalid-vector-db', { kind: 'ai-sqlite', - embedding: { ...MINILM_MODEL, revision: 'main' }, + embedding: { model: MINILM_MODEL, revision: 'main' }, credentials: { url: ':memory:' } }), /Only model and directory are supported/