diff --git a/migrations/0001_extensions.sql b/migrations/0001_extensions.sql index 6ef2928..ee17c77 100644 --- a/migrations/0001_extensions.sql +++ b/migrations/0001_extensions.sql @@ -1,4 +1,8 @@ +-- Own Postgres schema for every knowledge-engine table. Host control-plane +-- tables stay in public; this package never collides with or adopts them. +CREATE SCHEMA IF NOT EXISTS "knowledge"; + -- pgvector powers the dense retrieval channel. Per-model --- "knowledge_embedding_" vector tables are created at runtime by the +-- "knowledge"."embedding_" vector tables are created at runtime by the -- embed-model activation path (dimensionality varies by model), not here. CREATE EXTENSION IF NOT EXISTS vector; diff --git a/migrations/0002_knowledge_document.sql b/migrations/0002_knowledge_document.sql index 2f67aed..320649d 100644 --- a/migrations/0002_knowledge_document.sql +++ b/migrations/0002_knowledge_document.sql @@ -5,7 +5,8 @@ -- -- Identity/ACL lives here: tenant_id scopes every query; visibility_mode + -- the two visibility_* columns are the self-contained ACL the caller passes. -CREATE TABLE IF NOT EXISTS "knowledge_document" ( +-- tenant_id is plain text (no FK into control-plane tables). +CREATE TABLE IF NOT EXISTS "knowledge"."document" ( "id" text PRIMARY KEY, "tenant_id" text NOT NULL, "kind" text NOT NULL, @@ -18,13 +19,13 @@ CREATE TABLE IF NOT EXISTS "knowledge_document" ( "attributes" jsonb NOT NULL DEFAULT '{}', "created_at" timestamp NOT NULL DEFAULT now(), "last_seen_at" timestamp NOT NULL DEFAULT now(), - CONSTRAINT "knowledge_document_visibility_mode_check" CHECK ( + CONSTRAINT "document_visibility_mode_check" CHECK ( "visibility_mode" IN ('tenant', 'principals', 'source_acl', 'private') ) ); -CREATE UNIQUE INDEX IF NOT EXISTS "knowledge_document_tenant_adapter_external_ref_uniq" - ON "knowledge_document" ( +CREATE UNIQUE INDEX IF NOT EXISTS "document_tenant_adapter_external_ref_uniq" + ON "knowledge"."document" ( "tenant_id", "adapter", "external_ref" diff --git a/migrations/0003_knowledge_version.sql b/migrations/0003_knowledge_version.sql index af1f324..8c4ad06 100644 --- a/migrations/0003_knowledge_version.sql +++ b/migrations/0003_knowledge_version.sql @@ -1,4 +1,4 @@ --- The versioned body of a knowledge_document. version is a monotonic int per +-- The versioned body of a knowledge.document. version is a monotonic int per -- document; status tracks the version/supersede/deprecate lifecycle. Chunks -- belong to a version_id and are never reused across versions. -- @@ -6,10 +6,10 @@ -- (human/agent/system/adapter). The authority_* columns are a per-version -- snapshot of the corroboration signals computed at capture time (never -- recomputed retroactively) and consumed as a rank prior at search time. -CREATE TABLE IF NOT EXISTS "knowledge_version" ( +CREATE TABLE IF NOT EXISTS "knowledge"."version" ( "id" text PRIMARY KEY, "tenant_id" text NOT NULL, - "document_id" text NOT NULL REFERENCES "knowledge_document" ("id") ON DELETE CASCADE, + "document_id" text NOT NULL REFERENCES "knowledge"."document" ("id") ON DELETE CASCADE, "version" integer NOT NULL, "supersedes_version_id" text, "status" text NOT NULL DEFAULT 'active', @@ -25,25 +25,25 @@ CREATE TABLE IF NOT EXISTS "knowledge_version" ( "actor_count" integer NOT NULL DEFAULT 1, "has_social_signal" boolean NOT NULL DEFAULT false, "source_class" text NOT NULL DEFAULT 'native', - CONSTRAINT "knowledge_version_status_check" CHECK ( + CONSTRAINT "version_status_check" CHECK ( "status" IN ('active', 'superseded', 'deprecated', 'archived', 'tombstoned') ), - CONSTRAINT "knowledge_version_created_by_kind_check" CHECK ( + CONSTRAINT "version_created_by_kind_check" CHECK ( "created_by_kind" IN ('human', 'agent', 'system', 'adapter') ), - CONSTRAINT "knowledge_version_source_class_check" CHECK ( + CONSTRAINT "version_source_class_check" CHECK ( "source_class" IN ('native', 'thread', 'channel', 'call', 'record') ) ); -CREATE UNIQUE INDEX IF NOT EXISTS "knowledge_version_document_version_uniq" - ON "knowledge_version" ( +CREATE UNIQUE INDEX IF NOT EXISTS "version_document_version_uniq" + ON "knowledge"."version" ( "document_id", "version" ); -CREATE INDEX IF NOT EXISTS "knowledge_version_document_status_idx" - ON "knowledge_version" ( +CREATE INDEX IF NOT EXISTS "version_document_status_idx" + ON "knowledge"."version" ( "document_id", "status" ); diff --git a/migrations/0004_knowledge_chunk.sql b/migrations/0004_knowledge_chunk.sql index e3644cc..e460e42 100644 --- a/migrations/0004_knowledge_chunk.sql +++ b/migrations/0004_knowledge_chunk.sql @@ -1,12 +1,12 @@ --- An ordered slice of a knowledge_version's text, keyed by (version_id, +-- An ordered slice of a knowledge.version's text, keyed by (version_id, -- ordinal). text_fts is a generated tsvector for the lexical search channel — -- no vector column here (per-model embedding tables are created separately, -- since dimensionality varies by model). -CREATE TABLE IF NOT EXISTS "knowledge_chunk" ( +CREATE TABLE IF NOT EXISTS "knowledge"."chunk" ( "id" text PRIMARY KEY, "tenant_id" text NOT NULL, - "version_id" text NOT NULL REFERENCES "knowledge_version" ("id") ON DELETE CASCADE, - "document_id" text NOT NULL REFERENCES "knowledge_document" ("id") ON DELETE CASCADE, + "version_id" text NOT NULL REFERENCES "knowledge"."version" ("id") ON DELETE CASCADE, + "document_id" text NOT NULL REFERENCES "knowledge"."document" ("id") ON DELETE CASCADE, "ordinal" integer NOT NULL, "text" text NOT NULL, "role" text, @@ -14,11 +14,11 @@ CREATE TABLE IF NOT EXISTS "knowledge_chunk" ( "text_fts" tsvector GENERATED ALWAYS AS (to_tsvector('{{FTS_LANGUAGE}}', "text")) STORED ); -CREATE UNIQUE INDEX IF NOT EXISTS "knowledge_chunk_version_ordinal_uniq" - ON "knowledge_chunk" ( +CREATE UNIQUE INDEX IF NOT EXISTS "chunk_version_ordinal_uniq" + ON "knowledge"."chunk" ( "version_id", "ordinal" ); -CREATE INDEX IF NOT EXISTS "knowledge_chunk_text_fts_idx" - ON "knowledge_chunk" USING GIN ("text_fts"); +CREATE INDEX IF NOT EXISTS "chunk_text_fts_idx" + ON "knowledge"."chunk" USING GIN ("text_fts"); diff --git a/migrations/0005_knowledge_entity.sql b/migrations/0005_knowledge_entity.sql index 20189cc..b6ae492 100644 --- a/migrations/0005_knowledge_entity.sql +++ b/migrations/0005_knowledge_entity.sql @@ -1,6 +1,6 @@ -- A real-world thing (person, org, deal, ...) a document or chunk mentions. -- Identity keys only (email, domain, ...) — not another copy of chunk text. -CREATE TABLE IF NOT EXISTS "knowledge_entity" ( +CREATE TABLE IF NOT EXISTS "knowledge"."entity" ( "id" text PRIMARY KEY, "tenant_id" text NOT NULL, "kind" text NOT NULL, @@ -9,8 +9,8 @@ CREATE TABLE IF NOT EXISTS "knowledge_entity" ( "updated_at" timestamp NOT NULL DEFAULT now() ); -CREATE INDEX IF NOT EXISTS "knowledge_entity_tenant_kind_idx" - ON "knowledge_entity" ( +CREATE INDEX IF NOT EXISTS "entity_tenant_kind_idx" + ON "knowledge"."entity" ( "tenant_id", "kind" ); diff --git a/migrations/0006_knowledge_edge.sql b/migrations/0006_knowledge_edge.sql index 10f7708..ecd862e 100644 --- a/migrations/0006_knowledge_edge.sql +++ b/migrations/0006_knowledge_edge.sql @@ -2,7 +2,7 @@ -- (e.g. a principal) — never another copy of chunk text. This is the "series -- of relations" every record can carry; relation-following ingestion writes -- `mentions` edges here. -CREATE TABLE IF NOT EXISTS "knowledge_edge" ( +CREATE TABLE IF NOT EXISTS "knowledge"."edge" ( "id" text PRIMARY KEY, "tenant_id" text NOT NULL, "rel" text NOT NULL, @@ -11,26 +11,26 @@ CREATE TABLE IF NOT EXISTS "knowledge_edge" ( "to_type" text NOT NULL, "to_ref" text NOT NULL, "created_at" timestamp NOT NULL DEFAULT now(), - CONSTRAINT "knowledge_edge_rel_check" CHECK ( + CONSTRAINT "edge_rel_check" CHECK ( "rel" IN ('about', 'produced_by', 'links', 'parent', 'mentions', 'waiting_on') ), - CONSTRAINT "knowledge_edge_from_type_check" CHECK ( + CONSTRAINT "edge_from_type_check" CHECK ( "from_type" IN ('document', 'entity', 'native') ), - CONSTRAINT "knowledge_edge_to_type_check" CHECK ( + CONSTRAINT "edge_to_type_check" CHECK ( "to_type" IN ('document', 'entity', 'native') ) ); -CREATE INDEX IF NOT EXISTS "knowledge_edge_from_idx" - ON "knowledge_edge" ( +CREATE INDEX IF NOT EXISTS "edge_from_idx" + ON "knowledge"."edge" ( "tenant_id", "from_type", "from_ref" ); -CREATE INDEX IF NOT EXISTS "knowledge_edge_to_idx" - ON "knowledge_edge" ( +CREATE INDEX IF NOT EXISTS "edge_to_idx" + ON "knowledge"."edge" ( "tenant_id", "to_type", "to_ref" diff --git a/migrations/0007_knowledge_embed_model.sql b/migrations/0007_knowledge_embed_model.sql index 637dc8f..315dd9e 100644 --- a/migrations/0007_knowledge_embed_model.sql +++ b/migrations/0007_knowledge_embed_model.sql @@ -1,7 +1,7 @@ -- Which embedding model is active per tenant, and the dims it was discovered --- at (never hard-coded). The per-model "knowledge_embedding_" vector +-- at (never hard-coded). The per-model "knowledge"."embedding_" vector -- tables are runtime-managed by the single guarded activation path, not here. -CREATE TABLE IF NOT EXISTS "knowledge_embed_model" ( +CREATE TABLE IF NOT EXISTS "knowledge"."embed_model" ( "id" text PRIMARY KEY, "tenant_id" text NOT NULL, "model_key" text NOT NULL, @@ -10,5 +10,5 @@ CREATE TABLE IF NOT EXISTS "knowledge_embed_model" ( "status" text NOT NULL DEFAULT 'active', "created_at" timestamp NOT NULL DEFAULT now(), "updated_at" timestamp NOT NULL DEFAULT now(), - CONSTRAINT "knowledge_embed_model_tenant_model_key_uniq" UNIQUE ("tenant_id", "model_key") + CONSTRAINT "embed_model_tenant_model_key_uniq" UNIQUE ("tenant_id", "model_key") ); diff --git a/migrations/0008_raw_capture.sql b/migrations/0008_raw_capture.sql index 8c802a2..d7ac68c 100644 --- a/migrations/0008_raw_capture.sql +++ b/migrations/0008_raw_capture.sql @@ -4,7 +4,7 @@ -- later replay can re-derive under a different config without re-fetching -- source. Append-only: rows are never updated or deleted by ingestion; dedupe -- on (tenant_id, source_hash) reuses the existing row. -CREATE TABLE IF NOT EXISTS "raw_capture" ( +CREATE TABLE IF NOT EXISTS "knowledge"."raw_capture" ( "id" text PRIMARY KEY, "tenant_id" text NOT NULL, "adapter" text NOT NULL, @@ -18,10 +18,10 @@ CREATE TABLE IF NOT EXISTS "raw_capture" ( ); CREATE UNIQUE INDEX IF NOT EXISTS "raw_capture_tenant_source_hash_uniq" - ON "raw_capture" ("tenant_id", "source_hash"); + ON "knowledge"."raw_capture" ("tenant_id", "source_hash"); CREATE INDEX IF NOT EXISTS "raw_capture_tenant_adapter_external_ref_idx" - ON "raw_capture" ("tenant_id", "adapter", "external_ref"); + ON "knowledge"."raw_capture" ("tenant_id", "adapter", "external_ref"); -ALTER TABLE "knowledge_version" - ADD COLUMN IF NOT EXISTS "raw_capture_id" text REFERENCES "raw_capture"("id"); +ALTER TABLE "knowledge"."version" + ADD COLUMN IF NOT EXISTS "raw_capture_id" text REFERENCES "knowledge"."raw_capture"("id"); diff --git a/migrations/0009_transform_pipeline.sql b/migrations/0009_transform_pipeline.sql index e341f45..05bdf4b 100644 --- a/migrations/0009_transform_pipeline.sql +++ b/migrations/0009_transform_pipeline.sql @@ -3,18 +3,18 @@ -- the immutable `raw_capture` rows under a NEW `generation` tag, without ever -- re-fetching source or touching the 'live' generation's versions. -ALTER TABLE "knowledge_version" +ALTER TABLE "knowledge"."version" ADD COLUMN IF NOT EXISTS "generation" text NOT NULL DEFAULT 'live'; -- Version numbering moves from per-document to per-(document, generation), -- so a replay generation can mint its own v1 alongside the live document's -- existing versions instead of colliding with them. -DROP INDEX IF EXISTS "knowledge_version_document_version_uniq"; +DROP INDEX IF EXISTS "knowledge"."version_document_version_uniq"; -CREATE UNIQUE INDEX IF NOT EXISTS "knowledge_version_document_generation_version_uniq" - ON "knowledge_version" ("document_id", "generation", "version"); +CREATE UNIQUE INDEX IF NOT EXISTS "version_document_generation_version_uniq" + ON "knowledge"."version" ("document_id", "generation", "version"); -CREATE TABLE IF NOT EXISTS "transform_config" ( +CREATE TABLE IF NOT EXISTS "knowledge"."transform_config" ( "id" text PRIMARY KEY, "tenant_id" text NOT NULL, "name" text NOT NULL, @@ -24,10 +24,10 @@ CREATE TABLE IF NOT EXISTS "transform_config" ( CONSTRAINT "transform_config_tenant_name_version_uniq" UNIQUE ("tenant_id", "name", "version") ); -CREATE TABLE IF NOT EXISTS "transform_run" ( +CREATE TABLE IF NOT EXISTS "knowledge"."transform_run" ( "id" text PRIMARY KEY, "tenant_id" text NOT NULL, - "config_id" text NOT NULL REFERENCES "transform_config" ("id"), + "config_id" text NOT NULL REFERENCES "knowledge"."transform_config" ("id"), "scope" jsonb NOT NULL DEFAULT '{}', "generation" text NOT NULL, "status" text NOT NULL DEFAULT 'running', @@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS "transform_run" ( -- One run mints exactly one generation; resolving a generation's tuning -- config at search time is a lookup on this uniqueness. CREATE UNIQUE INDEX IF NOT EXISTS "transform_run_generation_uniq" - ON "transform_run" ("generation"); + ON "knowledge"."transform_run" ("generation"); CREATE INDEX IF NOT EXISTS "transform_run_tenant_config_idx" - ON "transform_run" ("tenant_id", "config_id"); + ON "knowledge"."transform_run" ("tenant_id", "config_id"); diff --git a/src/core/embed-model-registry.test.ts b/src/core/embed-model-registry.test.ts index 7eba6ea..0b369c5 100644 --- a/src/core/embed-model-registry.test.ts +++ b/src/core/embed-model-registry.test.ts @@ -64,7 +64,7 @@ describe("computeModelKey / embeddingTableName", () => { expect(key).toMatch(/^[a-f0-9]{16}$/); const tableName = embeddingTableName(key); expect(tableName).toMatch(EMBED_TABLE_NAME_PATTERN); - expect(tableName).toBe(`knowledge_embedding_${key}`); + expect(tableName).toBe(`"knowledge"."embedding_${key}"`); }); it("rejects a key that would produce an invalid identifier", () => { @@ -133,7 +133,9 @@ describe("activateEmbedModel", () => { expect(result.dims).toBe(768); expect(result.tableName).toMatch(EMBED_TABLE_NAME_PATTERN); - const insertQuery = queries.find((q) => q.sql.includes("INSERT INTO knowledge_embed_model")); + const insertQuery = queries.find((q) => + q.sql.includes('INSERT INTO "knowledge"."embed_model"'), + ); expect(insertQuery).toBeDefined(); expect(insertQuery?.params).toContain("tenant-1"); expect(insertQuery?.params).toContain(768); @@ -141,16 +143,15 @@ describe("activateEmbedModel", () => { const createTableQuery = queries.find((q) => q.sql.includes("CREATE TABLE IF NOT EXISTS")); expect(createTableQuery?.sql).toContain(result.tableName); expect(createTableQuery?.sql).toContain("vector(768)"); + const bare = result.tableName.replace(/^"knowledge"\."|"$/g, ""); + expect(createTableQuery?.sql).toContain(`CONSTRAINT ${bare}_chunk_fk`); expect(createTableQuery?.sql).toContain( - `CONSTRAINT ${result.tableName}_chunk_fk`, - ); - expect(createTableQuery?.sql).toContain( - "FOREIGN KEY (chunk_id) REFERENCES knowledge_chunk (id) ON DELETE CASCADE", + 'FOREIGN KEY (chunk_id) REFERENCES "knowledge"."chunk" (id) ON DELETE CASCADE', ); const tenantIndexQuery = queries.find((q) => q.sql.includes("_tenant_chunk_idx")); expect(tenantIndexQuery?.sql).toBe( - `CREATE INDEX IF NOT EXISTS ${result.tableName}_tenant_chunk_idx ON ${result.tableName} (tenant_id, chunk_id)`, + `CREATE INDEX IF NOT EXISTS ${bare}_tenant_chunk_idx ON ${result.tableName} (tenant_id, chunk_id)`, ); const indexQuery = queries.find((q) => q.sql.includes("USING hnsw")); @@ -268,7 +269,7 @@ describe("resolveActiveEmbedTable", () => { }; const result = await resolveActiveEmbedTable(client, "tenant-1"); expect(result).toEqual({ - tableName: `knowledge_embedding_${modelKey}`, + tableName: `"knowledge"."embedding_${modelKey}"`, dims: 768, modelId: baseConfig.modelId, }); diff --git a/src/core/embed-model-registry.ts b/src/core/embed-model-registry.ts index f46be1a..e984747 100644 --- a/src/core/embed-model-registry.ts +++ b/src/core/embed-model-registry.ts @@ -31,9 +31,8 @@ export function cosineDistanceExpr( } // Dims are dynamic and discovered, never hard-coded — the dimension travels -// with exactly one artifact: the knowledge_embed_model.dims column, -// discovered here at configure time. Never resurrect an EMBED_DIM constant -// anywhere in this module. +// with exactly one artifact: knowledge.embed_model.dims, discovered here at +// configure time. Never resurrect an EMBED_DIM constant anywhere in this module. export const MIN_EMBED_DIMS = 64; // Upper bound is pgvector's halfvec index cap: above 4000 dims no index type // can serve the cosine query, so activation rejects the model outright @@ -69,19 +68,32 @@ export function computeModelKey(baseUrl: string, modelId: string): string { .slice(0, 16); } -export const EMBED_TABLE_NAME_PATTERN = /^knowledge_embedding_[a-f0-9]{16}$/; +/** Bare table identifier (no schema) for indexes/constraints. */ +export const EMBED_TABLE_BARE_PATTERN = /^embedding_[a-f0-9]{16}$/; + +/** + * Fully schema-qualified embedding table name for raw SQL interpolation. + * Tables live under the knowledge schema: "knowledge"."embedding_". + */ +export const EMBED_TABLE_NAME_PATTERN = + /^"knowledge"\."embedding_[a-f0-9]{16}"$/; // This is the only place in this module that ever interpolates a computed // identifier into raw SQL (see activateEmbedModel below) — a future change // must not add a second dynamic-DDL path. -export function embeddingTableName(modelKey: string): string { - const tableName = `knowledge_embedding_${modelKey}`; - if (!EMBED_TABLE_NAME_PATTERN.test(tableName)) { +export function embeddingTableBareName(modelKey: string): string { + const bare = `embedding_${modelKey}`; + if (!EMBED_TABLE_BARE_PATTERN.test(bare)) { throw new Error( - `Computed embedding table name "${tableName}" failed identifier validation`, + `Computed embedding table name "${bare}" failed identifier validation`, ); } - return tableName; + return bare; +} + +export function embeddingTableName(modelKey: string): string { + const bare = embeddingTableBareName(modelKey); + return `"knowledge"."${bare}"`; } // Minimal DB seam — this module takes no dependency on drizzle-orm/postgres @@ -108,9 +120,10 @@ export async function activateEmbedModel( const dims = await discoverModelDims(config, fetchImpl); const modelKey = computeModelKey(config.baseUrl, config.modelId); const tableName = embeddingTableName(modelKey); + const bare = embeddingTableBareName(modelKey); await client.query( - `INSERT INTO knowledge_embed_model (id, tenant_id, model_key, model_id, dims, status, created_at, updated_at) + `INSERT INTO "knowledge"."embed_model" (id, tenant_id, model_key, model_id, dims, status, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, 'active', now(), now()) ON CONFLICT (tenant_id, model_key) DO UPDATE SET model_id = EXCLUDED.model_id, dims = EXCLUDED.dims, updated_at = now()`, @@ -128,8 +141,8 @@ export async function activateEmbedModel( chunk_id text PRIMARY KEY, tenant_id text NOT NULL, embedding vector(${dims}), - CONSTRAINT ${tableName}_chunk_fk - FOREIGN KEY (chunk_id) REFERENCES knowledge_chunk (id) ON DELETE CASCADE + CONSTRAINT ${bare}_chunk_fk + FOREIGN KEY (chunk_id) REFERENCES "knowledge"."chunk" (id) ON DELETE CASCADE )`, [], ); @@ -138,11 +151,11 @@ export async function activateEmbedModel( // selects embedding). Runs on every activation so a pre-FK table still gets // the index. await client.query( - `CREATE INDEX IF NOT EXISTS ${tableName}_tenant_chunk_idx ON ${tableName} (tenant_id, chunk_id)`, + `CREATE INDEX IF NOT EXISTS ${bare}_tenant_chunk_idx ON ${tableName} (tenant_id, chunk_id)`, [], ); - const indexName = `${tableName}_hnsw_idx`; + const indexName = `${bare}_hnsw_idx`; if (dims <= VECTOR_INDEX_MAX_DIMS) { try { await client.query( @@ -190,7 +203,7 @@ export async function resolveActiveEmbedTable( tenantId: string, ): Promise { const rows = await client.query( - `SELECT model_key, model_id, dims FROM knowledge_embed_model + `SELECT model_key, model_id, dims FROM "knowledge"."embed_model" WHERE tenant_id = $1 AND status = 'active' ORDER BY updated_at DESC LIMIT 1`, diff --git a/src/core/fts-language.ts b/src/core/fts-language.ts index cbc3ff1..d4202fc 100644 --- a/src/core/fts-language.ts +++ b/src/core/fts-language.ts @@ -72,14 +72,14 @@ export function createFtsVerification( function rebuildColumnRecipe(language: string): string { return ( ` BEGIN;\n` + - ` DROP INDEX IF EXISTS knowledge_chunk_text_fts_idx;\n` + - ` ALTER TABLE knowledge_chunk DROP COLUMN text_fts;\n` + - ` ALTER TABLE knowledge_chunk ADD COLUMN text_fts tsvector\n` + + ` DROP INDEX IF EXISTS "knowledge"."chunk_text_fts_idx";\n` + + ` ALTER TABLE "knowledge"."chunk" DROP COLUMN text_fts;\n` + + ` ALTER TABLE "knowledge"."chunk" ADD COLUMN text_fts tsvector\n` + ` GENERATED ALWAYS AS (to_tsvector('${language}', "text")) STORED;\n` + ` COMMIT;\n\n` + ` -- Separate statement/connection — CANNOT run inside the transaction\n` + ` -- above, or any transaction block, ever:\n` + - ` CREATE INDEX CONCURRENTLY knowledge_chunk_text_fts_idx ON knowledge_chunk USING gin (text_fts);\n\n` + + ` CREATE INDEX CONCURRENTLY "knowledge"."chunk_text_fts_idx" ON "knowledge"."chunk" USING gin (text_fts);\n\n` + `Both ALTER TABLE statements take an ACCESS EXCLUSIVE lock and rewrite the table ` + `(DROP COLUMN then re-adding a STORED generated column forces a full rewrite) — ` + `expect a stall on this table for the duration on a populated database; run during a maintenance window.` @@ -92,7 +92,7 @@ export interface FtsVerifySqlClient { /** * Enforce the invariant the env var alone cannot: the language baked into - * knowledge_chunk.text_fts (read back from the catalog — the authoritative + * knowledge.chunk.text_fts (read back from the catalog — the authoritative * record of what the DDL actually applied) must equal the configured one, * and the configured one must be an installed text search config. Throws * with a rebuild instruction on mismatch. Run at startup (the migration @@ -116,12 +116,14 @@ export async function verifyFtsLanguage( `SELECT pg_get_expr(d.adbin, d.adrelid) AS expr FROM pg_attrdef d JOIN pg_attribute a ON a.attrelid = d.adrelid AND a.attnum = d.adnum - WHERE d.adrelid = 'knowledge_chunk'::regclass AND a.attname = 'text_fts'`, + WHERE d.adrelid = '"knowledge"."chunk"'::regclass AND a.attname = 'text_fts'`, [], ); const expr = rows[0]?.["expr"]; if (typeof expr !== "string") { - throw new Error("knowledge_chunk.text_fts has no generation expression — schema not migrated?"); + throw new Error( + 'knowledge.chunk.text_fts has no generation expression — schema not migrated?', + ); } // Only unqualified `pg_catalog` configs are supported: FTS_LANGUAGE_PATTERN // already refuses to configure a schema-qualified name, so a match here @@ -133,13 +135,13 @@ export async function verifyFtsLanguage( const match = APPLIED_REGCONFIG_RE.exec(expr); if (match === null) { throw new Error( - `Could not read the applied FTS language from knowledge_chunk.text_fts: ${expr}`, + `Could not read the applied FTS language from knowledge.chunk.text_fts: ${expr}`, ); } const [, schema, applied] = match; if (schema !== undefined) { throw new Error( - `knowledge_chunk.text_fts was built with the schema-qualified text search config "${schema}.${applied}", ` + + `knowledge.chunk.text_fts was built with the schema-qualified text search config "${schema}.${applied}", ` + `but FTS_LANGUAGE only supports unqualified pg_catalog configs. ` + `Either drop the schema qualification (move/alias the config into pg_catalog), or rebuild the column ` + `under an unqualified config name:\n\n${rebuildColumnRecipe(ftsLanguage)}`, @@ -147,7 +149,7 @@ export async function verifyFtsLanguage( } if (applied !== ftsLanguage) { throw new Error( - `FTS language mismatch: knowledge_chunk.text_fts was built with "${applied}" but the configuration says "${ftsLanguage}". ` + + `FTS language mismatch: knowledge.chunk.text_fts was built with "${applied}" but the configuration says "${ftsLanguage}". ` + `Search would silently stem queries differently than the index.\n\n` + `To rebuild the column under the new language:\n\n${rebuildColumnRecipe(ftsLanguage)}\n\n` + `Or, fix FTS_LANGUAGE back to "${applied}" instead.`, diff --git a/src/db/schema.test.ts b/src/db/schema.test.ts new file mode 100644 index 0000000..3508c5d --- /dev/null +++ b/src/db/schema.test.ts @@ -0,0 +1,61 @@ +/** + * CL-5233: all engine tables live under the knowledge Postgres schema. + */ +import { describe, expect, it } from "bun:test"; +import { getTableName } from "drizzle-orm"; +import { + KNOWLEDGE_SCHEMA, + knowledgeChunk, + knowledgeDocument, + knowledgeEdge, + knowledgeEmbedModel, + knowledgeEntity, + knowledgeSchema, + knowledgeVersion, + rawCapture, + transformConfig, + transformRun, +} from "./schema.ts"; + +const TABLES = [ + knowledgeDocument, + knowledgeVersion, + knowledgeChunk, + knowledgeEntity, + knowledgeEdge, + rawCapture, + knowledgeEmbedModel, + transformConfig, + transformRun, +] as const; + +describe("knowledge Postgres schema qualification (CL-5233)", () => { + it("exports KNOWLEDGE_SCHEMA = knowledge", () => { + expect(KNOWLEDGE_SCHEMA).toBe("knowledge"); + expect(knowledgeSchema.schemaName).toBe("knowledge"); + }); + + it("every table is registered under the knowledge schema", () => { + for (const table of TABLES) { + // drizzle Table internal schema key + const schemaName = (table as unknown as { [key: symbol]: unknown })[ + Symbol.for("drizzle:Schema") + ]; + expect(schemaName).toBe("knowledge"); + // bare names drop the redundant knowledge_ prefix + expect(getTableName(table)).not.toMatch(/^knowledge_/); + } + }); + + it("maps legacy knowledge_* names to short table names", () => { + expect(getTableName(knowledgeDocument)).toBe("document"); + expect(getTableName(knowledgeVersion)).toBe("version"); + expect(getTableName(knowledgeChunk)).toBe("chunk"); + expect(getTableName(knowledgeEntity)).toBe("entity"); + expect(getTableName(knowledgeEdge)).toBe("edge"); + expect(getTableName(knowledgeEmbedModel)).toBe("embed_model"); + expect(getTableName(rawCapture)).toBe("raw_capture"); + expect(getTableName(transformConfig)).toBe("transform_config"); + expect(getTableName(transformRun)).toBe("transform_run"); + }); +}); diff --git a/src/db/schema.ts b/src/db/schema.ts index e61c15f..b967f29 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -3,7 +3,7 @@ import { customType, integer, jsonb, - pgTable, + pgSchema, real, text, timestamp, @@ -11,6 +11,11 @@ import { index, } from "drizzle-orm/pg-core"; +/** Postgres schema owned by this package — never public. */ +export const KNOWLEDGE_SCHEMA = "knowledge"; + +export const knowledgeSchema = pgSchema(KNOWLEDGE_SCHEMA); + // No built-in `bytea` helper in drizzle-orm/pg-core; raw_capture.raw_bytes // holds non-textual raw payloads (binary source formats) as a Buffer. const bytea = customType<{ data: Buffer }>({ @@ -19,8 +24,8 @@ const bytea = customType<{ data: Buffer }>({ }, }); -export const knowledgeDocument = pgTable( - "knowledge_document", +export const knowledgeDocument = knowledgeSchema.table( + "document", { id: text("id").primaryKey(), tenantId: text("tenant_id").notNull(), @@ -36,7 +41,7 @@ export const knowledgeDocument = pgTable( lastSeenAt: timestamp("last_seen_at").notNull().defaultNow(), }, (t) => [ - uniqueIndex("knowledge_document_tenant_adapter_external_ref_uniq").on( + uniqueIndex("document_tenant_adapter_external_ref_uniq").on( t.tenantId, t.adapter, t.externalRef, @@ -44,8 +49,8 @@ export const knowledgeDocument = pgTable( ], ); -export const knowledgeVersion = pgTable( - "knowledge_version", +export const knowledgeVersion = knowledgeSchema.table( + "version", { id: text("id").primaryKey(), tenantId: text("tenant_id").notNull(), @@ -74,17 +79,17 @@ export const knowledgeVersion = pgTable( generation: text("generation").notNull().default("live"), }, (t) => [ - uniqueIndex("knowledge_version_document_generation_version_uniq").on( + uniqueIndex("version_document_generation_version_uniq").on( t.documentId, t.generation, t.version, ), - index("knowledge_version_document_status_idx").on(t.documentId, t.status), + index("version_document_status_idx").on(t.documentId, t.status), ], ); -export const knowledgeChunk = pgTable( - "knowledge_chunk", +export const knowledgeChunk = knowledgeSchema.table( + "chunk", { id: text("id").primaryKey(), tenantId: text("tenant_id").notNull(), @@ -100,15 +105,12 @@ export const knowledgeChunk = pgTable( createdAt: timestamp("created_at").notNull().defaultNow(), }, (t) => [ - uniqueIndex("knowledge_chunk_version_ordinal_uniq").on( - t.versionId, - t.ordinal, - ), + uniqueIndex("chunk_version_ordinal_uniq").on(t.versionId, t.ordinal), ], ); -export const knowledgeEntity = pgTable( - "knowledge_entity", +export const knowledgeEntity = knowledgeSchema.table( + "entity", { id: text("id").primaryKey(), tenantId: text("tenant_id").notNull(), @@ -117,11 +119,11 @@ export const knowledgeEntity = pgTable( createdAt: timestamp("created_at").notNull().defaultNow(), updatedAt: timestamp("updated_at").notNull().defaultNow(), }, - (t) => [index("knowledge_entity_tenant_kind_idx").on(t.tenantId, t.kind)], + (t) => [index("entity_tenant_kind_idx").on(t.tenantId, t.kind)], ); -export const knowledgeEdge = pgTable( - "knowledge_edge", +export const knowledgeEdge = knowledgeSchema.table( + "edge", { id: text("id").primaryKey(), tenantId: text("tenant_id").notNull(), @@ -133,8 +135,8 @@ export const knowledgeEdge = pgTable( createdAt: timestamp("created_at").notNull().defaultNow(), }, (t) => [ - index("knowledge_edge_from_idx").on(t.tenantId, t.fromType, t.fromRef), - index("knowledge_edge_to_idx").on(t.tenantId, t.toType, t.toRef), + index("edge_from_idx").on(t.tenantId, t.fromType, t.fromRef), + index("edge_to_idx").on(t.tenantId, t.toType, t.toRef), ], ); @@ -143,7 +145,7 @@ export const knowledgeEdge = pgTable( // different config without re-fetching source. Append-only; dedupe on // (tenantId, sourceHash) reuses the existing row instead of inserting a // duplicate. -export const rawCapture = pgTable( +export const rawCapture = knowledgeSchema.table( "raw_capture", { id: text("id").primaryKey(), @@ -170,7 +172,7 @@ export const rawCapture = pgTable( ], ); -export const knowledgeEmbedModel = pgTable("knowledge_embed_model", { +export const knowledgeEmbedModel = knowledgeSchema.table("embed_model", { id: text("id").primaryKey(), tenantId: text("tenant_id").notNull(), modelKey: text("model_key").notNull(), @@ -186,7 +188,7 @@ export const knowledgeEmbedModel = pgTable("knowledge_embed_model", { // + retrieval-boost config (see src/core/schemas/transform.ts); unique on // (tenant_id, name, version) so re-creating the same name mints a new // version rather than colliding. -export const transformConfig = pgTable( +export const transformConfig = knowledgeSchema.table( "transform_config", { id: text("id").primaryKey(), @@ -209,7 +211,7 @@ export const transformConfig = pgTable( // slice of raw_capture. `generation` is this run's id, written onto every // knowledge_version row it derives; unique so a generation always resolves // back to exactly one run (and therefore one config) at search time. -export const transformRun = pgTable( +export const transformRun = knowledgeSchema.table( "transform_run", { id: text("id").primaryKey(), diff --git a/src/migrations.ts b/src/migrations.ts index de3c56e..b77b7db 100644 --- a/src/migrations.ts +++ b/src/migrations.ts @@ -1,7 +1,8 @@ /** * Knowledge-plane (pgvector) schema migrations, callable by host apps. * Applies every migrations/*.sql in filename order, each in its own - * transaction, tracked in a `_migrations` ledger so re-runs are idempotent. + * transaction, tracked in knowledge._migrations so re-runs are idempotent + * and the ledger never collides with a host's public migration bookkeeping. */ import postgres from "postgres"; import { readdir, readFile } from "node:fs/promises"; @@ -12,6 +13,7 @@ import { verifyFtsLanguage, } from "./core/fts-language.ts"; import { createRawSqlClient } from "./core/embed-sql.ts"; +import { KNOWLEDGE_SCHEMA } from "./db/schema.ts"; const MIGRATIONS_DIR = join(import.meta.dir, "..", "migrations"); @@ -28,13 +30,20 @@ export async function runKnowledgeMigrations( ); const sql = postgres(databaseUrl, { max: 1 }); try { - await sql`CREATE TABLE IF NOT EXISTS "_migrations" ( + // Schema first so the ledger and every later migration can land inside it + // even when 0001 has not been applied yet (fresh DB) or was skipped. + await sql.unsafe( + `CREATE SCHEMA IF NOT EXISTS "${KNOWLEDGE_SCHEMA}"`, + ); + await sql.unsafe( + `CREATE TABLE IF NOT EXISTS "${KNOWLEDGE_SCHEMA}"."_migrations" ( "name" text PRIMARY KEY, "applied_at" timestamp NOT NULL DEFAULT now() - )`; - const appliedRows = await sql< - { name: string }[] - >`SELECT name FROM "_migrations"`; + )`, + ); + const appliedRows = (await sql.unsafe( + `SELECT name FROM "${KNOWLEDGE_SCHEMA}"."_migrations"`, + )) as unknown as { name: string }[]; const applied = new Set(appliedRows.map((row) => row.name)); const files = (await readdir(MIGRATIONS_DIR)) @@ -50,7 +59,10 @@ export async function runKnowledgeMigrations( const ddl = raw.replaceAll(FTS_LANGUAGE_TOKEN, ftsLanguage); await sql.begin(async (tx) => { await tx.unsafe(ddl); - await tx`INSERT INTO "_migrations" (name) VALUES (${file})`; + await tx.unsafe( + `INSERT INTO "${KNOWLEDGE_SCHEMA}"."_migrations" (name) VALUES ($1)`, + [file], + ); }); log(`applied ${file}`); } diff --git a/src/services/search.test.ts b/src/services/search.test.ts index 97384c5..958475a 100644 --- a/src/services/search.test.ts +++ b/src/services/search.test.ts @@ -65,6 +65,7 @@ describe("visibilityPredicateSql", () => { it("produces the identical predicate shape as the raw-SQL string used by the dense channel, for BOTH the with-principal and null-principal cases", () => { function normalize(rawSql: string): string { return rawSql + .replace(/"knowledge"\."document"\./g, "kd.") .replace(/"knowledge_document"\./g, "kd.") .replace(/"(\w+)"/g, "$1") .replace(/\$\d+/g, "$PARAM") @@ -295,7 +296,10 @@ describe("fetchDenseCandidates hnsw tuning", () => { unsafe: (sqlText: string) => { statements.push(sqlText); return Promise.resolve( - sqlText.includes("FROM knowledge_embed_model") ? [MODEL_ROW] : [], + sqlText.includes('FROM "knowledge"."embed_model"') || + sqlText.includes("FROM knowledge_embed_model") + ? [MODEL_ROW] + : [], ); }, begin: (cb: (t: FakeTx) => Promise) => cb(tx), diff --git a/src/services/search.ts b/src/services/search.ts index b648622..d4a93c5 100644 --- a/src/services/search.ts +++ b/src/services/search.ts @@ -410,9 +410,9 @@ export async function fetchLexicalCandidates( // Bound as a parameter and cast to regconfig — never spliced — and // required to match the language the generated column was built with // (verified against the catalog by runKnowledgeMigrations). - rankExpr = sql`ts_rank("knowledge_chunk"."text_fts", plainto_tsquery(${ftsLanguage}::regconfig, ${query}))`; + rankExpr = sql`ts_rank("knowledge"."chunk"."text_fts", plainto_tsquery(${ftsLanguage}::regconfig, ${query}))`; conditions.push( - sql`"knowledge_chunk"."text_fts" @@ plainto_tsquery(${ftsLanguage}::regconfig, ${query})`, + sql`"knowledge"."chunk"."text_fts" @@ plainto_tsquery(${ftsLanguage}::regconfig, ${query})`, ); } @@ -593,9 +593,9 @@ export async function fetchDenseCandidates( kv.created_by_kind AS created_by_kind, kv.generator_agent_id AS generator_agent_id, c.text AS snippet_text, kv.occurred_at AS occurred_at, kv.authority AS authority FROM ${activeTable.tableName} e - JOIN knowledge_chunk c ON c.id = e.chunk_id - JOIN knowledge_version kv ON kv.id = c.version_id - JOIN knowledge_document kd ON kd.id = c.document_id + JOIN "knowledge"."chunk" c ON c.id = e.chunk_id + JOIN "knowledge"."version" kv ON kv.id = c.version_id + JOIN "knowledge"."document" kd ON kd.id = c.document_id WHERE e.tenant_id = $1 AND c.tenant_id = $1 AND kv.status = 'active' AND kv.generation = ${generationParam} AND ${visibilitySql}