diff --git a/CHANGELOG.md b/CHANGELOG.md index 55fc7eb..ffee0b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ - The format is based on [Keep a Changelog](https://keepachangelog.com/). - This project adheres to [Semantic Versioning](https://semver.org/). +## Version 1.2.0 - tbd + +### Added + +- Add the `ai-sqlite` kind with a `VECTOR_EMBEDDING` function using ONNX Runtime and the `Xenova/all-MiniLM-L6-v2` model (384 dimensions) + - Downloads the pinned model revision on-demand from Hugging Face (~91MB), verifies its size and SHA-256, and caches it locally + - Supports both 3-parameter `(text, text_type, model_and_version)` and 4-parameter variants with `remote_source` + - Compatible with `SAP_GXY.20250407` and `SAP_GXY.20240715` model versions + - Synchronous execution suitable for SQLite user-defined functions + - **Note**: Produces 384-dimensional vectors (vs. 768 in SAP HANA) for efficiency in local development scenarios + + ## Version 1.1.0 - 2026-07-20 diff --git a/README.md b/README.md index 7a2de53..d572b53 100644 --- a/README.md +++ b/README.md @@ -205,6 +205,59 @@ resources: type: org.cloudfoundry.managed-service ``` +### 3. Local Vector Embeddings with SQLite + +The `ai-sqlite` database kind extends `@cap-js/sqlite` with local semantic embeddings using an ONNX model. + +#### Usage + +Install the optional runtime dependencies: + +```sh +npm add @cap-js/sqlite onnxruntime-node@1.20.1 +``` + +`ai-sqlite` currently requires exactly `onnxruntime-node` 1.20.1 because synchronous SQLite functions need a version-specific native runtime API. + +Select `ai-sqlite` for the database service: + +```json +{ + "cds": { + "requires": { + "db": "ai-sqlite" + } + } +} +``` + +The HANA-compatible SQL function can then be used in CQL: + +```js +SELECT.from('Books').columns` + VECTOR_EMBEDDING(title, 'DOCUMENT', 'SAP_GXY.20250407') as embedding +`; +``` + +**Parameters:** +- `text` - Text to embed (`NULL` remains `NULL`; empty text returns a zero vector) +- `text_type` - Type of text, e.g., `'DOCUMENT'` (currently informational) +- `model_and_version` - Model identifier, e.g., `'SAP_GXY.20250407'` or `'SAP_GXY.20240715'` + +**Returns:** +- JSON stringified array of embedding values (384 dimensions) + +**Features:** +- **Initialization**: The ONNX model is loaded when the `ai-sqlite` service starts +- **Verified cache**: The pinned model revision is cached by default below the user's data directory; set `CDS_AI_MODEL_CACHE` to use a pre-provisioned cache root +- **Deterministic**: Same input always produces same output +- **Normalized vectors**: All embeddings are L2-normalized +- **Semantic similarity**: Embeddings capture text meaning for similarity search + +**Error Handling:** +- Starting `ai-sqlite` fails if the ONNX model cannot be initialized +- Downloads are time-limited and accepted only when their expected size and SHA-256 match +- Throws if embedding generation fails ## Test the plugin locally diff --git a/lib/sqlite/AISQLiteService.js b/lib/sqlite/AISQLiteService.js index caff1e4..120476c 100644 --- a/lib/sqlite/AISQLiteService.js +++ b/lib/sqlite/AISQLiteService.js @@ -1,17 +1,22 @@ import SQLiteService from '@cap-js/sqlite'; +import { initializeEmbedding, vector_embedding } from '../vector_embedding/index.js'; export default class AISQLiteService extends SQLiteService { - init() { - // add this.xyz here + async init() { + await initializeEmbedding(); return super.init(); } get factory() { const factory = super.factory; - factory._create = factory.create; + const create = factory.create; factory.create = async (tenant) => { - const dbc = await factory._create(tenant); - // add dbc.xyz here + const dbc = await create(tenant); + const embedding = (input, textType, modelAndVersion) => + input == null ? null : vector_embedding(String(input), textType, modelAndVersion); + const deterministic = { deterministic: true }; + dbc.function('VECTOR_EMBEDDING', { ...deterministic, varargs: true }, embedding); + dbc.function('VECTOR_EMBEDDING', deterministic, embedding); return dbc; }; return factory; diff --git a/lib/vector_embedding/InferenceSession.js b/lib/vector_embedding/InferenceSession.js new file mode 100644 index 0000000..0ea71be --- /dev/null +++ b/lib/vector_embedding/InferenceSession.js @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +// +// Synchronous counterpart to onnxruntime-node's session handler. SQLite user +// defined functions cannot await the public asynchronous InferenceSession API. +import { createRequire } from 'module'; + +const require = createRequire(import.meta.url); +const SUPPORTED_ONNX_RUNTIME_VERSION = '1.20.1'; +const runtimeVersion = require('onnxruntime-node/package.json').version; + +if (runtimeVersion !== SUPPORTED_ONNX_RUNTIME_VERSION) { + throw new Error( + `Unsupported onnxruntime-node version ${runtimeVersion}; @cap-js/ai requires ${SUPPORTED_ONNX_RUNTIME_VERSION} because its synchronous SQLite integration uses the runtime's private native API.` + ); +} + +const ort = require('onnxruntime-node'); +const binding = require('onnxruntime-node/dist/binding.js').binding; + +class InferenceSession { + constructor(handler) { + this.handler = handler; + } + + run(feeds) { + if ( + typeof feeds !== 'object' || + feeds === null || + feeds instanceof ort.Tensor || + Array.isArray(feeds) + ) { + throw new TypeError( + "'feeds' must be an object that uses input names as keys and tensors as values." + ); + } + + for (const name of this.handler.inputNames) { + if (feeds[name] === undefined) throw new Error(`input '${name}' is missing in 'feeds'.`); + } + + const fetches = Object.fromEntries(this.handler.outputNames.map((name) => [name, null])); + const results = this.handler.run(feeds, fetches, {}); + const output = {}; + + for (const key in results) { + const result = results[key]; + output[key] = + result instanceof ort.Tensor + ? result + : new ort.Tensor(result.type, result.data, result.dims); + } + + return output; + } + + static async create(pathOrBuffer) { + if (typeof pathOrBuffer !== 'string' && !(pathOrBuffer instanceof Uint8Array)) { + throw new TypeError('Expected an ONNX model path or Uint8Array'); + } + return new InferenceSession(new SynchronousSessionHandler(pathOrBuffer)); + } +} + +class SynchronousSessionHandler { + constructor(pathOrBuffer) { + this.session = new binding.InferenceSession(); + if (typeof pathOrBuffer === 'string') { + this.session.loadModel(pathOrBuffer, {}); + } else { + this.session.loadModel( + pathOrBuffer.buffer, + pathOrBuffer.byteOffset, + pathOrBuffer.byteLength, + {} + ); + } + this.inputNames = this.session.inputNames; + this.outputNames = this.session.outputNames; + } + + run(feeds, fetches, options) { + return this.session.run(feeds, fetches, options); + } + + async dispose() { + this.session.dispose(); + } +} + +const { Tensor } = ort; + +export { InferenceSession, Tensor }; diff --git a/lib/vector_embedding/embedding.js b/lib/vector_embedding/embedding.js new file mode 100644 index 0000000..cb35081 --- /dev/null +++ b/lib/vector_embedding/embedding.js @@ -0,0 +1,210 @@ +import os from 'os'; +import path from 'path'; +import { Tensor } from './InferenceSession.js'; +import { + downloadModelIfNeeded, + loadModelAndTokenizer, + preTokenize, + wordPieceTokenize, + validateTokenIds +} from './model-utils.js'; + +const MODEL = { + repository: 'Xenova/all-MiniLM-L6-v2', + revision: '751bff37182d3f1213fa05d7196b954e230abad9', + files: [ + { + name: 'model.onnx', + path: 'onnx/model.onnx', + size: 90387606, + sha256: '759c3cd2b7fe7e93933ad23c4c9181b7396442a2ed746ec7c1d46192c469c46e' + }, + { + name: 'tokenizer.json', + path: 'tokenizer.json', + size: 711661, + sha256: 'da0e79933b9ed51798a3ae27893d3c5fa4a201126cef75586296df9b4d2c62a0' + } + ] +}; + +/** + * Main tokenization function that combines all steps + */ +function wordPieceTokenizer(text, tokenizer) { + const unkToken = '[UNK]'; + const clsToken = '[CLS]'; + const sepToken = '[SEP]'; + const { vocab, maxLength, normalizer } = tokenizer; + + const clsId = vocab.get(clsToken) ?? 101; + const sepId = vocab.get(sepToken) ?? 102; + const unkId = vocab.get(unkToken) ?? 100; + + if (typeof clsId !== 'number' || typeof sepId !== 'number' || typeof unkId !== 'number') { + throw new Error('Special tokens must have numeric IDs'); + } + + const preTokens = preTokenize(text, normalizer); + + const tokens = [clsToken]; + const ids = [clsId]; + + for (const preToken of preTokens) { + const wordPieceTokens = wordPieceTokenize(preToken, vocab, unkToken); + + for (const wpToken of wordPieceTokens) { + const tokenId = vocab.get(wpToken) ?? unkId; + tokens.push(wpToken); + ids.push(tokenId); + } + } + + tokens.push(sepToken); + ids.push(sepId); + + if (tokens.length <= maxLength) return [{ tokens, ids }]; + + // Keep each chunk within the limit embedded in the pinned tokenizer. + const maxContentLength = maxLength - 2; + const chunks = []; + const contentTokens = tokens.slice(1, -1); + const contentIds = ids.slice(1, -1); + + for (let i = 0; i < contentTokens.length; i += maxContentLength) { + const chunkTokens = [clsToken, ...contentTokens.slice(i, i + maxContentLength), sepToken]; + const chunkIds = [clsId, ...contentIds.slice(i, i + maxContentLength), sepId]; + + chunks.push({ + tokens: chunkTokens, + ids: chunkIds + }); + } + + return chunks; +} + +/** + * Process embeddings for multiple chunks and combine them + */ +function processChunkedEmbeddings(chunks, session) { + const embeddings = []; + + for (const chunk of chunks) { + const { ids } = chunk; + const validIds = validateTokenIds(ids); + + const inputIds = new BigInt64Array(validIds.map((i) => BigInt(i))); + const attentionMask = new BigInt64Array(validIds.length).fill(BigInt(1)); + const tokenTypeIds = new BigInt64Array(validIds.length).fill(BigInt(0)); + + const inputTensor = new Tensor('int64', inputIds, [1, validIds.length]); + const attentionTensor = new Tensor('int64', attentionMask, [1, validIds.length]); + const tokenTypeTensor = new Tensor('int64', tokenTypeIds, [1, validIds.length]); + + const feeds = { + input_ids: inputTensor, + attention_mask: attentionTensor, + token_type_ids: tokenTypeTensor + }; + + const results = session.run(feeds); + const lastHiddenState = results['last_hidden_state']; + if (!lastHiddenState) + throw new Error( + `ONNX model output 'last_hidden_state' not found. Available outputs: ${Object.keys(results).join(', ')}` + ); + const [, sequenceLength, hiddenSize] = lastHiddenState.dims; + const embeddingData = lastHiddenState.data; + + // Apply mean pooling across the sequence dimension + const pooledEmbedding = new Float32Array(hiddenSize); + for (let i = 0; i < hiddenSize; i++) { + let sum = 0; + for (let j = 0; j < sequenceLength; j++) { + sum += embeddingData[j * hiddenSize + i]; + } + pooledEmbedding[i] = sum / sequenceLength; + } + + embeddings.push(pooledEmbedding); + } + + // If multiple chunks, average the embeddings + if (embeddings.length === 1) return embeddings[0]; + + const hiddenSize = embeddings[0].length; + const avgEmbedding = new Float32Array(hiddenSize); + + for (let i = 0; i < hiddenSize; i++) { + let sum = 0; + for (const embedding of embeddings) { + sum += embedding[i]; + } + avgEmbedding[i] = sum / embeddings.length; + } + + return avgEmbedding; +} + +let session = null; +let tokenizer = null; + +async function createSession() { + const modelDir = getModelDir(); + await downloadModelIfNeeded(modelDir, MODEL); + ({ session, tokenizer } = await loadModelAndTokenizer(modelDir)); +} + +function embedding(text) { + if (!session || !tokenizer) + throw new Error( + 'Embedding session not initialized. Call createSession() before using embedding().' + ); + const chunks = wordPieceTokenizer(text, tokenizer); + const vector = normalizeEmbedding(processChunkedEmbeddings(chunks, session)); + + const chunkObj = { content: text }; + return Object.defineProperty(chunkObj, 'embedding', { + value: vector, + writable: true, + configurable: true, + enumerable: false + }); + + function normalizeEmbedding(embedding) { + let norm = 0; + for (let i = 0; i < embedding.length; i++) { + norm += embedding[i] * embedding[i]; + } + norm = Math.sqrt(norm); + if (norm === 0) return embedding; // Guard against division by zero + for (let i = 0; i < embedding.length; i++) { + embedding[i] = embedding[i] / norm; + } + return embedding; + } +} + +/** + * Get the platform-specific data directory for the application + * @param {string} appName - The application name (defaults to 'semantic-search') + * @returns {string} The full path to the data directory + */ +function getDataDir(appName = 'semantic-search') { + const home = os.homedir(); + const dir = + os.platform() === 'win32' + ? process.env.LOCALAPPDATA || process.env.APPDATA || path.join(home, 'AppData', 'Local') + : process.env.XDG_DATA_HOME || path.join(home, '.local', 'share'); + + return path.join(dir, appName); +} + +function getModelDir() { + const cacheRoot = process.env.CDS_AI_MODEL_CACHE || path.join(getDataDir(), 'models'); + return path.join(cacheRoot, MODEL.repository.replace('/', '_'), MODEL.revision); +} + +export default embedding; +export { embedding, createSession, wordPieceTokenizer }; diff --git a/lib/vector_embedding/index.js b/lib/vector_embedding/index.js new file mode 100644 index 0000000..edfa0b1 --- /dev/null +++ b/lib/vector_embedding/index.js @@ -0,0 +1,52 @@ +import cds from '@sap/cds'; + +const LOG = cds.log('@cap-js/ai'); + +let embeddingModule; +let initialization; + +async function initializeEmbedding() { + if (embeddingModule) return embeddingModule; + + initialization ??= import('./embedding.js') + .then(async (module) => { + await module.createSession(); + LOG.info('Vector embedding ONNX model initialized'); + return (embeddingModule = module); + }) + .catch((error) => { + initialization = undefined; + throw error; + }); + + return initialization; +} + +const model_dimensions = { + 'SAP_GXY.20250407': 384, + 'SAP_GXY.20240715': 384 +}; + +/** + * Synchronous wrapper for vector embedding function. + * Generates embeddings using ONNX model. + * The model is initialized automatically when this module is imported. + * + * @param {string} text - Text to embed + * @param {string} text_type - Type of text (e.g., 'DOCUMENT') + * @param {string} model_and_version - Model identifier (e.g., 'SAP_GXY.20250407') + * @returns {string} JSON stringified array of embedding values + * @throws {Error} If embedding module failed to initialize or generation fails + */ +function vector_embedding(text, text_type, model_and_version) { + if (!embeddingModule) { + throw new Error('Embedding module is not initialized'); + } + + if (text) { + return JSON.stringify(Array.from(embeddingModule.embedding(text).embedding)); + } + return JSON.stringify(new Array(model_dimensions[model_and_version] ?? 384).fill(0)); +} + +export { initializeEmbedding, vector_embedding }; diff --git a/lib/vector_embedding/model-utils.js b/lib/vector_embedding/model-utils.js new file mode 100644 index 0000000..da2bb61 --- /dev/null +++ b/lib/vector_embedding/model-utils.js @@ -0,0 +1,241 @@ +import { createHash, randomUUID } from 'crypto'; +import { createReadStream } from 'fs'; +import fs from 'fs/promises'; +import path from 'path'; +import { InferenceSession } from './InferenceSession.js'; + +const DOWNLOAD_TIMEOUT_MS = 5 * 60 * 1000; + +async function sha256(filePath) { + const hash = createHash('sha256'); + for await (const chunk of createReadStream(filePath)) hash.update(chunk); + return hash.digest('hex'); +} + +async function isValidFile(filePath, file) { + try { + const stat = await fs.stat(filePath); + return stat.isFile() && stat.size === file.size && (await sha256(filePath)) === file.sha256; + } catch (error) { + if (error.code === 'ENOENT') return false; + throw error; + } +} + +async function downloadFile(url, outputPath, file, options = {}) { + const { fetchImpl = globalThis.fetch, timeoutMs = DOWNLOAD_TIMEOUT_MS } = options; + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + const temporaryPath = `${outputPath}.${process.pid}.${randomUUID()}.tmp`; + let handle; + + try { + const response = await fetchImpl(url, { signal: controller.signal }); + if (!response.ok) { + throw new Error( + `Failed to download ${url}, status ${response.status} (${response.statusText})` + ); + } + if (!response.body) throw new Error(`Failed to download ${url}: response has no body`); + + const contentLength = Number(response.headers.get('content-length')); + if (Number.isFinite(contentLength) && contentLength > file.size) { + throw new Error(`Refusing ${url}: response exceeds the expected ${file.size} bytes`); + } + + handle = await fs.open(temporaryPath, 'wx', 0o600); + const hash = createHash('sha256'); + let bytesWritten = 0; + + for await (const value of response.body) { + const chunk = Buffer.from(value); + bytesWritten += chunk.byteLength; + if (bytesWritten > file.size) { + throw new Error(`Refusing ${url}: response exceeds the expected ${file.size} bytes`); + } + hash.update(chunk); + await handle.writeFile(chunk); + } + + await handle.sync(); + await handle.close(); + handle = undefined; + + if (bytesWritten !== file.size) { + throw new Error(`Invalid size for ${url}: expected ${file.size}, received ${bytesWritten}`); + } + const digest = hash.digest('hex'); + if (digest !== file.sha256) { + throw new Error(`Invalid SHA-256 for ${url}: expected ${file.sha256}, received ${digest}`); + } + + try { + await fs.rename(temporaryPath, outputPath); + } catch (error) { + if (error.code !== 'EEXIST' && error.code !== 'EPERM') throw error; + if (await isValidFile(outputPath, file)) await fs.unlink(temporaryPath); + else { + await fs.unlink(outputPath).catch(() => {}); + await fs.rename(temporaryPath, outputPath); + } + } + } catch (error) { + if (error.name === 'AbortError') { + throw new Error(`Timed out after ${timeoutMs} ms while downloading ${url}`, { cause: error }); + } + throw error; + } finally { + clearTimeout(timeout); + await handle?.close().catch(() => {}); + await fs.unlink(temporaryPath).catch(() => {}); + } +} + +async function downloadModelIfNeeded(modelDir, model, options) { + await fs.mkdir(modelDir, { recursive: true }); + + for (const file of model.files) { + const filePath = path.join(modelDir, file.name); + // eslint-disable-next-line no-await-in-loop + if (await isValidFile(filePath, file)) continue; + + const url = `https://huggingface.co/${model.repository}/resolve/${model.revision}/${file.path}`; + // Files are downloaded serially to avoid multiplying startup bandwidth and memory usage. + // eslint-disable-next-line no-await-in-loop + await downloadFile(url, filePath, file, options); + } +} + +async function loadModelAndTokenizer(modelDir) { + const modelPath = path.join(modelDir, 'model.onnx'); + const tokenizerPath = path.join(modelDir, 'tokenizer.json'); + const tokenizerJson = JSON.parse(await fs.readFile(tokenizerPath, 'utf8')); + + if (!tokenizerJson.model?.vocab) { + throw new Error('Invalid tokenizer structure: missing model.vocab'); + } + + const vocab = new Map(); + for (const [token, id] of Object.entries(tokenizerJson.model.vocab)) { + if (Number.isSafeInteger(id) && id >= 0) vocab.set(token, id); + } + + const maxLength = tokenizerJson.truncation?.max_length; + if (!Number.isSafeInteger(maxLength) || maxLength < 2) { + throw new Error('Invalid tokenizer structure: missing truncation.max_length'); + } + + const session = await InferenceSession.create(modelPath); + return { + session, + tokenizer: { + vocab, + maxLength, + normalizer: tokenizerJson.normalizer ?? {} + } + }; +} + +function preTokenize(text, normalizer = {}) { + const { + clean_text: cleanText = true, + handle_chinese_chars: handleChineseChars = true, + lowercase = true, + strip_accents: configuredStripAccents + } = normalizer; + const stripAccents = configuredStripAccents ?? lowercase; + let normalized = String(text); + + if (cleanText) { + normalized = Array.from(normalized, (character) => { + if (/\s/u.test(character)) return ' '; + if (character.codePointAt(0) === 0 || character.codePointAt(0) === 0xfffd) return ''; + if (/[\p{Cc}\p{Cf}]/u.test(character)) return ''; + return character; + }).join(''); + } + + if (handleChineseChars) { + normalized = Array.from(normalized, (character) => + isChineseCharacter(character.codePointAt(0)) ? ` ${character} ` : character + ).join(''); + } + + const output = []; + for (let token of normalized.trim().split(/\s+/u)) { + if (!token) continue; + if (lowercase) token = token.toLowerCase(); + if (stripAccents) token = token.normalize('NFD').replace(/\p{M}/gu, ''); + + let current = ''; + for (const character of token) { + if (/\p{P}/u.test(character)) { + if (current) output.push(current); + output.push(character); + current = ''; + } else current += character; + } + if (current) output.push(current); + } + return output; +} + +function isChineseCharacter(codePoint) { + return ( + (codePoint >= 0x4e00 && codePoint <= 0x9fff) || + (codePoint >= 0x3400 && codePoint <= 0x4dbf) || + (codePoint >= 0x20000 && codePoint <= 0x2a6df) || + (codePoint >= 0x2a700 && codePoint <= 0x2b73f) || + (codePoint >= 0x2b740 && codePoint <= 0x2b81f) || + (codePoint >= 0x2b820 && codePoint <= 0x2ceaf) || + (codePoint >= 0xf900 && codePoint <= 0xfaff) || + (codePoint >= 0x2f800 && codePoint <= 0x2fa1f) + ); +} + +function wordPieceTokenize(token, vocab, unkToken = '[UNK]', maxInputCharsPerWord = 100) { + if (Array.from(token).length > maxInputCharsPerWord) return [unkToken]; + + const outputTokens = []; + let start = 0; + while (start < token.length) { + let end = token.length; + let currentSubstring = null; + + while (start < end) { + let substring = token.substring(start, end); + if (start > 0) substring = '##' + substring; + if (vocab.has(substring)) { + currentSubstring = substring; + break; + } + end -= 1; + } + + if (currentSubstring === null) return [unkToken]; + + outputTokens.push(currentSubstring); + start = end; + } + + return outputTokens; +} + +function validateTokenIds(ids) { + ids.forEach((id) => { + if (!Number.isSafeInteger(id) || id < 0) { + throw new Error(`Invalid token ID detected: ${id} (type: ${typeof id})`); + } + }); + return ids; +} + +export { + downloadFile, + downloadModelIfNeeded, + isValidFile, + loadModelAndTokenizer, + preTokenize, + wordPieceTokenize, + validateTokenIds +}; diff --git a/package.json b/package.json index bfb70e5..8df0da8 100644 --- a/package.json +++ b/package.json @@ -22,10 +22,17 @@ ], "devDependencies": { "@cap-js/cds-test": "^1", - "@cap-js/cds-types": "^0.16.0" + "@cap-js/cds-types": "^0.16.0", + "onnxruntime-node": "1.20.1" }, "peerDependencies": { - "@sap/cds": ">=9" + "@sap/cds": ">=9", + "onnxruntime-node": "1.20.1" + }, + "peerDependenciesMeta": { + "onnxruntime-node": { + "optional": true + } }, "engines": { "node": ">=20.0.0" diff --git a/tests/vector-unit.test.js b/tests/vector-unit.test.js new file mode 100644 index 0000000..76e30f7 --- /dev/null +++ b/tests/vector-unit.test.js @@ -0,0 +1,133 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, test } from 'node:test'; +import { wordPieceTokenizer } from '../lib/vector_embedding/embedding.js'; +import { downloadModelIfNeeded } from '../lib/vector_embedding/model-utils.js'; + +const temporaryDirectories = []; + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => fs.rm(directory, { recursive: true, force: true })) + ); +}); + +describe('BERT tokenizer', () => { + const tokenizer = { + maxLength: 128, + normalizer: { + clean_text: true, + handle_chinese_chars: true, + lowercase: true, + strip_accents: null + }, + vocab: new Map([ + ['[UNK]', 100], + ['[CLS]', 101], + ['[SEP]', 102], + ['hello', 200], + [',', 201], + ['cafe', 202], + ['中', 203], + ['文', 204], + ['token', 205] + ]) + }; + + test('applies BERT accent, punctuation, and Chinese character normalization', () => { + const [chunk] = wordPieceTokenizer('Héllo, café中文', tokenizer); + + assert.deepEqual(chunk.tokens, ['[CLS]', 'hello', ',', 'cafe', '中', '文', '[SEP]']); + assert.deepEqual(chunk.ids, [101, 200, 201, 202, 203, 204, 102]); + }); + + test('uses the tokenizer model limit without an off-by-one', () => { + const chunks = wordPieceTokenizer(new Array(130).fill('token').join(' '), tokenizer); + + assert.deepEqual( + chunks.map(({ ids }) => ids.length), + [128, 6] + ); + assert.ok(chunks.every(({ ids }) => ids.length <= tokenizer.maxLength)); + }); +}); + +describe('model download', () => { + test('uses a pinned revision and atomically caches a verified file', async () => { + const directory = await createTemporaryDirectory(); + const content = Buffer.from('verified model fixture'); + const model = fixtureModel(content); + const requestedUrls = []; + const fetchImpl = async (url) => { + requestedUrls.push(url); + const body = new ReadableStream({ + start(controller) { + controller.enqueue(content.subarray(0, 5)); + controller.enqueue(content.subarray(5)); + controller.close(); + } + }); + return new Response(body, { + headers: { 'content-length': String(content.length) } + }); + }; + + await downloadModelIfNeeded(directory, model, { fetchImpl }); + await downloadModelIfNeeded(directory, model, { fetchImpl }); + + assert.deepEqual(requestedUrls, [ + 'https://huggingface.co/example/model/resolve/deadbeef/model.onnx' + ]); + assert.deepEqual(await fs.readFile(path.join(directory, 'model.onnx')), content); + assert.deepEqual(await fs.readdir(directory), ['model.onnx']); + }); + + test('rejects oversized content without exposing a partial cache file', async () => { + const directory = await createTemporaryDirectory(); + const content = Buffer.from('expected'); + const model = fixtureModel(content); + const fetchImpl = async () => new Response(Buffer.concat([content, Buffer.from('extra')])); + + await assert.rejects( + downloadModelIfNeeded(directory, model, { fetchImpl }), + /exceeds the expected 8 bytes/ + ); + assert.deepEqual(await fs.readdir(directory), []); + }); + + test('rejects content that does not match the pinned checksum', async () => { + const directory = await createTemporaryDirectory(); + const content = Buffer.from('expected'); + const model = fixtureModel(content); + const fetchImpl = async () => new Response(Buffer.from('tampered')); + + await assert.rejects(downloadModelIfNeeded(directory, model, { fetchImpl }), /Invalid SHA-256/); + assert.deepEqual(await fs.readdir(directory), []); + }); +}); + +async function createTemporaryDirectory() { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'cap-ai-model-')); + temporaryDirectories.push(directory); + return directory; +} + +function fixtureModel(content) { + return { + repository: 'example/model', + revision: 'deadbeef', + files: [ + { + name: 'model.onnx', + path: 'model.onnx', + size: content.length, + sha256: createHash('sha256').update(content).digest('hex') + } + ] + }; +} diff --git a/tests/vector.test.js b/tests/vector.test.js new file mode 100644 index 0000000..0c20ab3 --- /dev/null +++ b/tests/vector.test.js @@ -0,0 +1,153 @@ +import { after, before, describe, test } from 'node:test'; +import assert from 'node:assert'; +import cds from '@sap/cds'; +import { initializeEmbedding, vector_embedding } from '../lib/vector_embedding/index.js'; + +before(initializeEmbedding); + +describe('Vector embedding function (standalone)', () => { + describe('vector_embedding', () => { + test('computes embedding with ONNX model', async () => { + const result = vector_embedding('Hello world', 'DOCUMENT', 'SAP_GXY.20250407'); + + const embedding = JSON.parse(result); + assert.ok(Array.isArray(embedding), 'Embedding should be an array'); + assert.strictEqual(embedding.length, 384, 'Embedding should have 384 dimensions'); + + // Check that values are floats in reasonable range + embedding.forEach((val, idx) => { + assert.strictEqual(typeof val, 'number', `Value at index ${idx} should be a number`); + assert.ok(Math.abs(val) <= 1, `Value at index ${idx} should be normalized (-1 to 1)`); + }); + }); + + test('deterministic - same input produces same output', async () => { + const e1 = vector_embedding('test text', 'DOCUMENT', 'SAP_GXY.20250407'); + const e2 = vector_embedding('test text', 'DOCUMENT', 'SAP_GXY.20250407'); + + assert.strictEqual(e1, e2, 'Same input should produce identical embeddings'); + }); + + test('different inputs produce different outputs', async () => { + const e1 = vector_embedding('hello world', 'DOCUMENT', 'SAP_GXY.20250407'); + const e2 = vector_embedding('goodbye world', 'DOCUMENT', 'SAP_GXY.20250407'); + + assert.notStrictEqual(e1, e2, 'Different inputs should produce different embeddings'); + }); + + test('semantically similar sentences produce similar vectors', async () => { + const e1 = vector_embedding('I love programming', 'DOCUMENT', 'SAP_GXY.20250407'); + const e2 = vector_embedding('I enjoy coding', 'DOCUMENT', 'SAP_GXY.20250407'); + + const v1 = JSON.parse(e1); + const v2 = JSON.parse(e2); + + const similarity = cosineSimilarity(v1, v2); + assert.ok( + similarity > 0.8, + `Semantically similar sentences should have high cosine similarity (got ${similarity.toFixed(3)})` + ); + }); + + test('semantically different sentences are far apart in vector space', async () => { + const e1 = vector_embedding('The cat sat on the mat', 'DOCUMENT', 'SAP_GXY.20250407'); + const e2 = vector_embedding('Quantum physics is fascinating', 'DOCUMENT', 'SAP_GXY.20250407'); + + const v1 = JSON.parse(e1); + const v2 = JSON.parse(e2); + + const similarity = cosineSimilarity(v1, v2); + assert.ok( + similarity < 0.1, + `Semantically different sentences should have low cosine similarity (got ${similarity.toFixed(3)})` + ); + }); + + test('handles empty text', async () => { + const result = vector_embedding('', 'DOCUMENT', 'SAP_GXY.20250407'); + + const embedding = JSON.parse(result); + assert.ok(Array.isArray(embedding), 'Empty text should return zero vector'); + assert.strictEqual(embedding.length, 384, 'Should have 384 dimensions'); + assert.ok( + embedding.every((v) => v === 0), + 'Empty text should return all zeros' + ); + }); + + test('handles null text', async () => { + const result = vector_embedding(null, 'DOCUMENT', 'SAP_GXY.20250407'); + + const embedding = JSON.parse(result); + assert.ok(Array.isArray(embedding), 'Null text should return zero vector'); + assert.strictEqual(embedding.length, 384, 'Should have 384 dimensions'); + assert.ok( + embedding.every((v) => v === 0), + 'Null text should return all zeros' + ); + }); + + test('uses correct dimensions for different models', async () => { + const result1 = vector_embedding('test', 'DOCUMENT', 'SAP_GXY.20250407'); + const embedding1 = JSON.parse(result1); + assert.strictEqual(embedding1.length, 384, 'SAP_GXY.20250407 should have 384 dimensions'); + + const result2 = vector_embedding('test', 'DOCUMENT', 'SAP_GXY.20240715'); + const embedding2 = JSON.parse(result2); + assert.strictEqual(embedding2.length, 384, 'SAP_GXY.20240715 should have 384 dimensions'); + + const result3 = vector_embedding('test', 'DOCUMENT', 'unknown_model'); + const embedding3 = JSON.parse(result3); + assert.strictEqual(embedding3.length, 384, 'Unknown model should default to 384 dimensions'); + }); + }); +}); + +describe('ai-sqlite integration', () => { + let db; + + before(async () => { + db = await cds.connect.to('vector-db', { + kind: 'ai-sqlite', + credentials: { url: ':memory:' } + }); + }); + + after(async () => { + await db?.disconnect(); + }); + + test('registers VECTOR_EMBEDDING for three and four arguments', async () => { + const [row] = await db.run(`SELECT + VECTOR_EMBEDDING('Hello world', 'DOCUMENT', 'SAP_GXY.20250407') AS local, + VECTOR_EMBEDDING('Hello world', 'DOCUMENT', 'SAP_GXY.20250407', 'remote') AS remote`); + + assert.strictEqual(JSON.parse(row.local).length, 384); + assert.strictEqual(row.remote, row.local); + }); + + test('preserves SQL null semantics', async () => { + const [row] = await db.run( + `SELECT VECTOR_EMBEDDING(NULL, 'DOCUMENT', 'SAP_GXY.20250407') AS embedding` + ); + + assert.strictEqual(row.embedding, null); + }); +}); + +// Helper function to calculate cosine similarity between two vectors +function cosineSimilarity(a, b) { + if (a.length !== b.length) throw new Error('Vectors must have the same length'); + + let dotProduct = 0; + let normA = 0; + let normB = 0; + + for (let i = 0; i < a.length; i++) { + dotProduct += a[i] * b[i]; + normA += a[i] * a[i]; + normB += b[i] * b[i]; + } + + return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)); +}