From 4c26ddfeede99cc4e9103491c9a91d07900facb9 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 3 Aug 2026 15:39:18 -0700 Subject: [PATCH] Rename the knowledge plane to green add/find/ask/recent (CL-5224) Public KnowledgePlane surface is now add, find, ask, recent with principalId and tenantId only. add returns {documentId}, accepts content XOR file via TextExtractor, and share sugar that maps onto the existing ACL path. find defaults green (no evidence unless asked). Old capture/search/timeline names are gone from the package export; HTTP routes still use legacy paths and grant verbs until CL-5225. --- src/index.ts | 14 +- src/knowledge.test.ts | 434 ++++++++++++++++++++++++++++++++++++-- src/knowledge.ts | 430 +++++++++++++++++++++++++++++-------- src/routes/capture.ts | 12 +- src/routes/routes.test.ts | 46 ++-- src/routes/search.ts | 27 ++- src/routes/timeline.ts | 2 +- 7 files changed, 832 insertions(+), 133 deletions(-) diff --git a/src/index.ts b/src/index.ts index a4c978a..50cad03 100644 --- a/src/index.ts +++ b/src/index.ts @@ -28,7 +28,7 @@ export type { EngineConfig } from "./config.ts"; export { RerankConfigError } from "./core/rerank-client.ts"; // Knowledge plane // -// `createKnowledgePlane` is exported so a host can capture or search outside a +// `createKnowledgePlane` is exported so a host can add or find outside a // request — a CLI seeder, a batch ingester, or a test — without standing up a // Hono app just to get a plane. Callers acting on behalf of a user are // responsible for the capability check `requireGrant` would have performed; see @@ -39,15 +39,21 @@ export type { AskCitation, AskResult, ChatMessage, + FindItem, + FindResult, Generate, HybridSearchResult, + KnowledgeAddParams, + KnowledgeAddResult, KnowledgeAskParams, - KnowledgeCaptureParams, + KnowledgeFindParams, KnowledgeIdentity, KnowledgePlane, KnowledgePlaneOptions, - KnowledgeSearchParams, + KnowledgeRecentParams, + KnowledgeShare, SearchHit, + TextExtractor, TimelineEvent, VisibilitySpec, } from "./knowledge.ts"; @@ -83,7 +89,7 @@ export type MountKnowledgeEngineOptions = { */ grants: GrantConfig; /** - * How `ask()` reaches a model. Omit if this host only captures and searches; + * How `ask()` reaches a model. Omit if this host only adds and finds; * `ask()` then fails with a 501 naming what is missing. * * The engine owns no generation client on purpose — Interchange's diff --git a/src/knowledge.test.ts b/src/knowledge.test.ts index c53c8cb..76481ee 100644 --- a/src/knowledge.test.ts +++ b/src/knowledge.test.ts @@ -2,10 +2,12 @@ * Plane construction, ACL wiring, and ask() coverage for knowledge.ts. * * - Construction: rerank maxDocChars validation runs in createKnowledgePlane. - * - Search wiring: acl.test.ts covers blockedDocumentIds itself; the post-filter + * - Find wiring: acl.test.ts covers blockedDocumentIds itself; the post-filter * call site is pinned here so deleting or inverting it fails the suite. - * - ask(): grant check, missing generate (501 before search), allow path, + * - ask(): grant check, missing generate (501 before find), allow path, * synthesizeAnswer grounding. + * - add(): documentId return, content/file XOR, share → ACL mapping. + * - find/recent: limit bounds, evidence default omit. */ import { afterAll, @@ -26,10 +28,14 @@ import { KnowledgeNotPermittedError, synthesizeAnswer, type ChatMessage, + type FindItem, + type KnowledgeAddParams, + type TextExtractor, } from "./knowledge.ts"; import type { KnowledgeConfig } from "./mount-config.ts"; import * as realDb from "./db/client.ts"; import * as realSearch from "./services/search.ts"; +import * as realCapture from "./services/capture.ts"; import type { HybridSearchResult } from "./services/search.ts"; const PRINCIPAL = "p1"; @@ -40,7 +46,7 @@ type DocAclRow = { attributes: { acl_block?: unknown }; }; -/** Satisfies createFtsVerification → createRawSqlClient(sql).unsafe on the search path. */ +/** Satisfies createFtsVerification → createRawSqlClient(sql).unsafe on the find path. */ const ENGLISH_FTS_EXPR = "to_tsvector('english'::regconfig, text)"; function grant(action: string): GrantRule { @@ -102,6 +108,17 @@ function hit(overrides: Partial | string = {}): SearchHit { }; } +function findItemFromHit(h: SearchHit): FindItem { + return { + documentId: h.document_id, + title: h.title, + snippet: h.snippet, + score: h.score, + kind: h.kind, + citation: h.citation, + }; +} + const wiringConfig: KnowledgeConfig = { knowledge: { databaseUrl: "postgres://localhost:5432/nonexistent-test-db", @@ -156,7 +173,7 @@ describe("createKnowledgePlane — construction validation", () => { it("throws RerankConfigError when maxDocChars overflows a known TEI model", () => { // Proves validateRerankConfig runs inside createKnowledgePlane (not only // mountKnowledgeEngine): a standalone plane with a bad override must fail - // construction, not silently degrade on every later search. + // construction, not silently degrade on every later find. expect(() => createKnowledgePlane( baseConfig({ @@ -170,7 +187,7 @@ describe("createKnowledgePlane — construction validation", () => { }); }); -describe("createKnowledgePlane.search — ACL post-filter wiring", () => { +describe("createKnowledgePlane.find — ACL post-filter wiring", () => { const hybridSearch = mock((): Promise => Promise.resolve({ hits: [hit("d-blocked"), hit("d-open")], @@ -241,14 +258,15 @@ describe("createKnowledgePlane.search — ACL post-filter wiring", () => { `./knowledge.ts?wiring-blocked=${Date.now()}` ); const plane = makePlane(wiringConfig); - const result = await plane.search({ + const result = await plane.find({ tenantId: TENANT, principalId: PRINCIPAL, query: "q", + includeEvidence: true, }); expect(hybridSearch).toHaveBeenCalled(); - expect(result.hits.map((h: SearchHit) => h.document_id)).toEqual([ + expect(result.items.map((i: FindItem) => i.documentId)).toEqual([ "d-open", ]); expect(result.evidence).toBe("strong"); @@ -270,7 +288,7 @@ describe("createKnowledgePlane.search — ACL post-filter wiring", () => { `./knowledge.ts?wiring-kinds=${Date.now()}` ); const plane = makePlane(wiringConfig); - await plane.search({ + await plane.find({ tenantId: TENANT, principalId: PRINCIPAL, query: "q", @@ -313,17 +331,389 @@ describe("createKnowledgePlane.search — ACL post-filter wiring", () => { `./knowledge.ts?wiring-unreadable=${Date.now()}` ); const plane = makePlane(wiringConfig); - const result = await plane.search({ + const result = await plane.find({ tenantId: TENANT, principalId: PRINCIPAL, query: "q", + includeEvidence: true, }); - expect(result.hits).toEqual([]); + expect(result.items).toEqual([]); expect(result.evidence).toBe("none"); await plane.close(); }); + + it("omits evidence by default and includes it when includeEvidence is true", async () => { + hybridSearch.mockClear(); + hybridSearch.mockImplementation(() => + Promise.resolve({ + hits: [hit("d-open")], + evidence: "strong" as const, + degraded: ["dense_unavailable" as const], + }), + ); + sql.mockClear(); + sql.mockImplementation(() => + Promise.resolve([{ id: "d-open", attributes: {} }]), + ); + + const { createKnowledgePlane: makePlane } = await import( + `./knowledge.ts?wiring-evidence=${Date.now()}` + ); + const plane = makePlane(wiringConfig); + + const without = await plane.find({ + tenantId: TENANT, + principalId: PRINCIPAL, + query: "q", + }); + expect(without.items).toHaveLength(1); + expect(without.evidence).toBeUndefined(); + expect(without.degraded).toBeUndefined(); + + const withEv = await plane.find({ + tenantId: TENANT, + principalId: PRINCIPAL, + query: "q", + includeEvidence: true, + }); + expect(withEv.items).toHaveLength(1); + expect(withEv.evidence).toBe("strong"); + expect(withEv.degraded).toEqual(["dense_unavailable"]); + + await plane.close(); + }); +}); + +describe("find/recent — limit bounds", () => { + // These throw before any DB work, so a nonexistent URL is fine. + it("find rejects limit below 1", async () => { + const plane = createKnowledgePlane(wiringConfig); + try { + await plane.find({ + tenantId: TENANT, + principalId: PRINCIPAL, + query: "q", + limit: 0, + }); + throw new Error("expected find() to reject"); + } catch (err) { + expect(err).toBeInstanceOf(KnowledgeError); + expect((err as KnowledgeError).status).toBe(400); + expect((err as KnowledgeError).message).toContain("limit"); + } + }); + + it("find rejects limit above 50", async () => { + const plane = createKnowledgePlane(wiringConfig); + try { + await plane.find({ + tenantId: TENANT, + principalId: PRINCIPAL, + query: "q", + limit: 51, + }); + throw new Error("expected find() to reject"); + } catch (err) { + expect(err).toBeInstanceOf(KnowledgeError); + expect((err as KnowledgeError).status).toBe(400); + } + }); + + it("recent rejects limit above 100", async () => { + const plane = createKnowledgePlane(wiringConfig); + try { + await plane.recent({ + tenantId: TENANT, + principalId: PRINCIPAL, + limit: 101, + }); + throw new Error("expected recent() to reject"); + } catch (err) { + expect(err).toBeInstanceOf(KnowledgeError); + expect((err as KnowledgeError).status).toBe(400); + } + }); + + it("recent rejects limit below 1", async () => { + const plane = createKnowledgePlane(wiringConfig); + try { + await plane.recent({ + tenantId: TENANT, + principalId: PRINCIPAL, + limit: 0, + }); + throw new Error("expected recent() to reject"); + } catch (err) { + expect(err).toBeInstanceOf(KnowledgeError); + expect((err as KnowledgeError).status).toBe(400); + } + }); +}); + +describe("add() — documentId, content/file XOR, share", () => { + type CaptureDocResult = { + status: "captured" | "noop"; + documentId: string; + versionId: string; + chunks: number; + }; + const captureDocument = mock( + (): Promise => + Promise.resolve({ + status: "captured", + documentId: "kdoc_test_1", + versionId: "kver_1", + chunks: 1, + }), + ); + + const sql = Object.assign(mock(() => Promise.resolve([])), { + end: mock(() => Promise.resolve()), + unsafe: mock((sqlText: string) => ftsUnsafe(sqlText)), + }); + + beforeAll(() => { + mock.module("./db/client.ts", () => ({ + ...realDb, + createDb: () => ({ db: {}, sql }), + })); + mock.module("./services/capture.ts", () => ({ + ...realCapture, + captureDocument, + })); + }); + + afterAll(() => { + mock.module("./db/client.ts", () => realDb); + mock.module("./services/capture.ts", () => realCapture); + }); + +async function freshPlane(opts?: { + textExtractor?: TextExtractor; + }) { + const { createKnowledgePlane: makePlane } = await import( + `./knowledge.ts?add-${Date.now()}-${Math.random()}` + ); + return makePlane(wiringConfig, undefined, opts ?? {}); + } + + /** Dynamic re-import yields a distinct KnowledgeError class; match by shape. */ + function expectKnowledgeError400(err: unknown, messagePart: string) { + expect(err).toBeInstanceOf(Error); + expect((err as Error).name).toBe("KnowledgeError"); + expect((err as { status: number }).status).toBe(400); + expect((err as Error).message).toContain(messagePart); + } + + it("returns documentId from captureDocument (captured status)", async () => { + captureDocument.mockClear(); + captureDocument.mockImplementation(() => + Promise.resolve({ + status: "captured" as const, + documentId: "kdoc_captured", + versionId: "kver_1", + chunks: 1, + }), + ); + const plane = await freshPlane(); + const result = await plane.add({ + tenantId: TENANT, + principalId: PRINCIPAL, + content: { title: "T", text: "body" }, + }); + expect(result).toEqual({ documentId: "kdoc_captured" }); + await plane.close(); + }); + + it("returns documentId on noop status too", async () => { + captureDocument.mockClear(); + captureDocument.mockImplementation(() => + Promise.resolve({ + status: "noop" as const, + documentId: "kdoc_noop", + versionId: "kver_1", + chunks: 0, + }), + ); + const plane = await freshPlane(); + const result = await plane.add({ + tenantId: TENANT, + principalId: PRINCIPAL, + content: { title: "T", text: "body" }, + }); + expect(result).toEqual({ documentId: "kdoc_noop" }); + await plane.close(); + }); + + it("rejects when neither content nor file is provided", async () => { + const plane = await freshPlane(); + try { + await plane.add({ + tenantId: TENANT, + principalId: PRINCIPAL, + } as KnowledgeAddParams); + throw new Error("expected add() to reject"); + } catch (err) { + expectKnowledgeError400(err, "content or file"); + } + await plane.close(); + }); + + it("rejects when both content and file are provided", async () => { + const plane = await freshPlane(); + try { + await plane.add({ + tenantId: TENANT, + principalId: PRINCIPAL, + content: { title: "T", text: "body" }, + file: { bytes: new Uint8Array([1]) }, + }); + throw new Error("expected add() to reject"); + } catch (err) { + expectKnowledgeError400(err, "content or file"); + } + await plane.close(); + }); + + it("rejects file without a textExtractor", async () => { + const plane = await freshPlane(); + try { + await plane.add({ + tenantId: TENANT, + principalId: PRINCIPAL, + file: { bytes: new Uint8Array([1]), filename: "a.pdf" }, + }); + throw new Error("expected add() to reject"); + } catch (err) { + expectKnowledgeError400(err, "textExtractor"); + } + await plane.close(); + }); + + it("extracts text via textExtractor when file is provided", async () => { + captureDocument.mockClear(); + captureDocument.mockImplementation(() => + Promise.resolve({ + status: "captured" as const, + documentId: "kdoc_file", + versionId: "kver_1", + chunks: 1, + }), + ); + const textExtractor: TextExtractor = { + extract: mock(() => + Promise.resolve({ text: "extracted body", title: "From Extractor" }), + ), + }; + const plane = await freshPlane({ textExtractor }); + const result = await plane.add({ + tenantId: TENANT, + principalId: PRINCIPAL, + file: { + bytes: new Uint8Array([1, 2, 3]), + mimeType: "application/pdf", + filename: "note.pdf", + }, + }); + expect(result.documentId).toBe("kdoc_file"); + expect(textExtractor.extract).toHaveBeenCalled(); + expect(captureDocument).toHaveBeenCalled(); + const call = captureDocument.mock.calls[0] as unknown as [ + unknown, + { document: { title: string; chunks: { text: string }[] } }, + ]; + expect(call[1].document.title).toBe("From Extractor"); + expect(call[1].document.chunks[0]?.text).toBe("extracted body"); + await plane.close(); + }); + + it("maps share private to visibility private with owner principalId", async () => { + captureDocument.mockClear(); + captureDocument.mockImplementation(() => + Promise.resolve({ + status: "captured" as const, + documentId: "kdoc_share", + versionId: "kver_1", + chunks: 1, + }), + ); + const plane = await freshPlane(); + await plane.add({ + tenantId: TENANT, + principalId: PRINCIPAL, + content: { title: "Private note", text: "secret" }, + share: { mode: "private", block: ["blocked-p"] }, + }); + const call = captureDocument.mock.calls[0] as unknown as [ + unknown, + { + document: { + visibility: { mode: string; principalIds?: string[] }; + attributes?: { acl_block?: string }; + }; + }, + ]; + expect(call[1].document.visibility).toEqual({ + mode: "private", + principalIds: [PRINCIPAL], + }); + expect(call[1].document.attributes?.acl_block).toBe( + JSON.stringify(["blocked-p"]), + ); + await plane.close(); + }); + + it("maps share principals and always includes the owner", async () => { + captureDocument.mockClear(); + captureDocument.mockImplementation(() => + Promise.resolve({ + status: "captured" as const, + documentId: "kdoc_principals", + versionId: "kver_1", + chunks: 1, + }), + ); + const plane = await freshPlane(); + await plane.add({ + tenantId: TENANT, + principalId: PRINCIPAL, + content: { title: "Shared", text: "body" }, + share: { mode: "principals", principalIds: ["alice", "bob"] }, + }); + const call = captureDocument.mock.calls[0] as unknown as [ + unknown, + { + document: { + visibility: { mode: string; principalIds?: string[] }; + }; + }, + ]; + expect(call[1].document.visibility.mode).toBe("principals"); + const ids = call[1].document.visibility.principalIds ?? []; + expect(ids).toContain(PRINCIPAL); + expect(ids).toContain("alice"); + expect(ids).toContain("bob"); + await plane.close(); + }); + + it("rejects share together with visibility", async () => { + const plane = await freshPlane(); + try { + await plane.add({ + tenantId: TENANT, + principalId: PRINCIPAL, + content: { title: "T", text: "body" }, + share: { mode: "tenant" }, + visibility: { mode: "private", principalIds: [PRINCIPAL] }, + }); + throw new Error("expected add() to reject"); + } catch (err) { + expectKnowledgeError400(err, "share or visibility"); + } + await plane.close(); + }); }); describe("ask() — grant check", () => { @@ -352,14 +742,14 @@ describe("ask() — grant check", () => { }); describe("ask() — missing generate", () => { - it("throws KnowledgeError 501 before search when generate is not wired", async () => { - // Pointed at a nonexistent DB: if search ran first this would surface a + it("throws KnowledgeError 501 before find when generate is not wired", async () => { + // Pointed at a nonexistent DB: if find ran first this would surface a // connection/driver error instead of the promised 501. const grants = { grantStore: createInMemoryGrantStore([grant("search")]), conditionRegistry: {}, }; -const plane = createKnowledgePlane(askConfig, grants); + const plane = createKnowledgePlane(askConfig, grants); try { await plane.ask({ tenantId: TENANT, principalId: PRINCIPAL, query: "q" }); throw new Error("expected ask() to reject"); @@ -372,7 +762,7 @@ const plane = createKnowledgePlane(askConfig, grants); }); describe("ask() — allow path", () => { - it("searches as the principal and synthesizes when grant allows and generate is wired", async () => { + it("finds as the principal and synthesizes when grant allows and generate is wired", async () => { const grants = { grantStore: createInMemoryGrantStore([grant("search")]), conditionRegistry: {}, @@ -384,11 +774,14 @@ describe("ask() — allow path", () => { expect(messages[1]?.content).toContain("the relevant snippet"); return Promise.resolve("Answer from context [1]."); }); -const plane = createKnowledgePlane(askConfig, grants, { generate }); - // Stub search so this unit test never needs a live Postgres. ask() looks - // up plane.search at call time, so reassignment is the wiring under test. - plane.search = mock(() => - Promise.resolve({ hits: [hit()], evidence: "strong" as const }), + const plane = createKnowledgePlane(askConfig, grants, { generate }); + // Stub find so this unit test never needs a live Postgres. ask() looks + // up plane.find at call time, so reassignment is the wiring under test. + plane.find = mock(() => + Promise.resolve({ + items: [findItemFromHit(hit())], + evidence: "strong" as const, + }), ); const result = await plane.ask({ @@ -397,10 +790,11 @@ const plane = createKnowledgePlane(askConfig, grants, { generate }); query: "what is the answer?", }); - expect(plane.search).toHaveBeenCalledWith({ + expect(plane.find).toHaveBeenCalledWith({ tenantId: TENANT, principalId: PRINCIPAL, query: "what is the answer?", + includeEvidence: true, }); expect(generate).toHaveBeenCalledTimes(1); expect(result.text).toBe("Answer from context [1]."); diff --git a/src/knowledge.ts b/src/knowledge.ts index c6f69c3..4976d12 100644 --- a/src/knowledge.ts +++ b/src/knowledge.ts @@ -1,6 +1,8 @@ /** * Knowledge plane backed by the engine's pgvector Postgres. Wraps the capture * and hybrid-search services directly — no HTTP hop. + * + * Green surface: add / find / ask / recent. */ import { authorize } from "@intx/authz"; @@ -19,15 +21,17 @@ import { KnowledgeSearchInputError, toRerankClientConfig, type HybridSearchResult, + DEFAULT_HYBRID_TOP_K, } from "./services/search.ts"; import { listTimelineEvents, type TimelineEvent, + DEFAULT_TIMELINE_LIMIT, } from "./services/timeline.ts"; import type { KnowledgeConfig } from "./mount-config.ts"; import type { GrantConfig } from "./routes/deps.ts"; -// Re-export so hosts typing plane.search() results don't reach into services/. +// Re-export so hosts typing plane results don't reach into services/. export type { HybridSearchResult } from "./services/search.ts"; export type { SearchHit } from "./core/schemas/search.ts"; export type { VisibilitySpec } from "./core/schemas/document.ts"; @@ -51,31 +55,54 @@ export type ChatMessage = { */ export type Generate = (messages: readonly ChatMessage[]) => Promise; +/** + * Optional host-supplied extractor for `add({ file })`. The engine never + * ships a PDF/OCR/vendor SDK — the host plugs one in when file ingest is needed. + */ +export type TextExtractor = { + extract(file: { + bytes: Uint8Array; + mimeType?: string; + filename?: string; + }): Promise<{ text: string; title?: string }>; +}; + export type KnowledgeIdentity = { principalId: string; tenantId: string; }; -export type KnowledgeSearchParams = KnowledgeIdentity & { +/** Green find limit bounds (stricter than hybridSearch's internal MAX_K). */ +export const FIND_LIMIT_MIN = 1; +export const FIND_LIMIT_MAX = 50; + +/** Green recent limit bounds (matches timeline service default/cap). */ +export const RECENT_LIMIT_MIN = 1; +export const RECENT_LIMIT_MAX = DEFAULT_TIMELINE_LIMIT; + +export type KnowledgeFindParams = KnowledgeIdentity & { query: string; - k?: number | undefined; + /** Max items to return (1–50). Default 8. */ + limit?: number; + /** When true, include evidence (and degraded if any). Default: omit. */ + includeEvidence?: boolean; /** * Narrows every retrieval channel to documents whose `kind` is one of * these — see `hybridSearch` in services/search.ts. Applied before fusion, * so a fused hit is always guaranteed to match. Unset or an empty array * both mean "no filter" (equivalent, not "match nothing"). */ - kinds?: string[] | undefined; + kinds?: string[]; /** * Same scoping as `kinds`, restricted to documents linked to one of these * entity ids. Unset or an empty array both mean "no filter". */ - entityIds?: string[] | undefined; + entityIds?: string[]; }; export type KnowledgeAskParams = KnowledgeIdentity & { query: string; - k?: number; + limit?: number; }; /** One source cited in an `ask()` answer, matched to its bracket in the text. */ @@ -101,19 +128,54 @@ export class KnowledgeNotPermittedError extends Error { } } -export type KnowledgeCaptureParams = KnowledgeIdentity & { - title: string; - text: string; +export type KnowledgeShare = + | { mode: "private" } + | { mode: "tenant" } + | { mode: "principals"; principalIds: string[] }; + +export type KnowledgeAddParams = KnowledgeIdentity & { + /** Exactly one of `content` or `file` is required. */ + content?: { title: string; text: string }; + file?: { + bytes: Uint8Array; + mimeType?: string; + filename?: string; + title?: string; + }; kind?: string; adapter?: string; externalRef?: string; + /** Direct escape hatch; mutually exclusive with `share`. */ visibility?: VisibilitySpec; - /** Principal ids blocked from seeing this doc (stored for read-path post-filter). */ + /** Direct escape hatch; with `share`, use `share.block` instead. */ blockPrincipalIds?: string[]; + /** + * Sugar for visibility + optional block list. + * Mutually exclusive with `visibility`. + */ + share?: KnowledgeShare & { block?: string[] }; attributes?: Record; }; -export type KnowledgeTimelineParams = KnowledgeIdentity & { +export type KnowledgeAddResult = { documentId: string }; + +export type FindItem = { + documentId: string; + title: string; + snippet: string; + score: number; + kind: string; + citation: SearchHit["citation"]; +}; + +export type FindResult = { + items: FindItem[]; + /** Only present when includeEvidence: true */ + evidence?: "strong" | "weak" | "none"; + degraded?: HybridSearchResult["degraded"]; +}; + +export type KnowledgeRecentParams = KnowledgeIdentity & { limit?: number; }; @@ -128,10 +190,10 @@ export class KnowledgeError extends Error { } export type KnowledgePlane = { - search(params: KnowledgeSearchParams): Promise; + find(params: KnowledgeFindParams): Promise; ask(params: KnowledgeAskParams): Promise; - capture(params: KnowledgeCaptureParams): Promise; - timeline(params: KnowledgeTimelineParams): Promise; + add(params: KnowledgeAddParams): Promise; + recent(params: KnowledgeRecentParams): Promise; close(): Promise; }; @@ -225,15 +287,126 @@ export async function synthesizeAnswer( } export type KnowledgePlaneOptions = { - /** Required for `ask()`; omit if the host only captures and searches. */ + /** Required for `ask()`; omit if the host only adds and finds. */ generate?: Generate; + /** Required for `add({ file })`; omit if the host only adds text content. */ + textExtractor?: TextExtractor; }; +function resolveFindLimit(limit: number | undefined): number { + if (limit === undefined) return DEFAULT_HYBRID_TOP_K; + if ( + typeof limit !== "number" || + !Number.isInteger(limit) || + limit < FIND_LIMIT_MIN || + limit > FIND_LIMIT_MAX + ) { + throw new KnowledgeError( + 400, + `limit must be an integer between ${FIND_LIMIT_MIN} and ${FIND_LIMIT_MAX}`, + ); + } + return limit; +} + +function resolveRecentLimit(limit: number | undefined): number | undefined { + if (limit === undefined) return undefined; + if ( + typeof limit !== "number" || + !Number.isInteger(limit) || + limit < RECENT_LIMIT_MIN || + limit > RECENT_LIMIT_MAX + ) { + throw new KnowledgeError( + 400, + `limit must be an integer between ${RECENT_LIMIT_MIN} and ${RECENT_LIMIT_MAX}`, + ); + } + return limit; +} + +function hitsToFindItems(hits: readonly SearchHit[]): FindItem[] { + return hits.map((h) => ({ + documentId: h.document_id, + title: h.title, + snippet: h.snippet, + score: h.score, + kind: h.kind, + citation: h.citation, + })); +} + +/** Map FindItems back to the minimal SearchHit shape synthesizeAnswer needs. */ +function findItemsToHits(items: readonly FindItem[]): SearchHit[] { + return items.map((item) => ({ + chunk_id: "", + document_id: item.documentId, + version: 0, + version_id: "", + status: "active" as const, + score: item.score, + title: item.title, + snippet: item.snippet, + kind: item.kind, + created_by_kind: "human" as const, + citation: item.citation, + entity_ids: [], + channels_matched: [], + })); +} + +function resolveShareAndVisibility(params: KnowledgeAddParams): { + visibility: VisibilitySpec; + blockPrincipalIds?: string[]; +} { + if (params.share !== undefined && params.visibility !== undefined) { + throw new KnowledgeError( + 400, + "provide share or visibility, not both", + ); + } + + if (params.share !== undefined) { + if (params.blockPrincipalIds !== undefined) { + throw new KnowledgeError( + 400, + "provide share.block or blockPrincipalIds, not both", + ); + } + const share = params.share; + let visibility: VisibilitySpec; + if (share.mode === "private") { + visibility = { + mode: "private", + principalIds: [params.principalId], + }; + } else if (share.mode === "tenant") { + visibility = { mode: "tenant" }; + } else { + const ids = new Set(share.principalIds); + ids.add(params.principalId); + visibility = { mode: "principals", principalIds: [...ids] }; + } + const block = share.block; + return { + visibility, + ...(block && block.length > 0 ? { blockPrincipalIds: block } : {}), + }; + } + + return { + visibility: params.visibility ?? { mode: "tenant" as const }, + ...(params.blockPrincipalIds !== undefined + ? { blockPrincipalIds: params.blockPrincipalIds } + : {}), + }; +} + /** * Build a knowledge plane. * * - `grants` is required for `ask()` (in-process capability check). Standalone - * capture/search callers may omit it — same as #8's out-of-band plane. + * add/find callers may omit it — same as #8's out-of-band plane. * - Rerank config is validated at construction (same as mount). */ export function createKnowledgePlane( @@ -242,7 +415,7 @@ export function createKnowledgePlane( options: KnowledgePlaneOptions = {}, ): KnowledgePlane { // Catch a chunk-size / reranker-limit mismatch at construction time, rather - // than silently on every search once the reranker starts rejecting batches. + // than silently on every find once the reranker starts rejecting batches. // Throws instead of warning: a mismatch means every rerank call for this // host WILL 413 and silently degrade to fused ranking, with no per-request // signal — a construction-time failure surfaces that once, loudly. @@ -271,7 +444,7 @@ export function createKnowledgePlane( // synchronous, so "before accepting traffic" becomes a memoized check // awaited by the first query. Read-only; migration stays a deploy step. // NOTE this is a lazy check, not a boot-time one: nothing forces it to run - // until the first real search()/capture() call, so a host that neither + // until the first real find()/add() call, so a host that neither // runs runKnowledgeMigrations itself nor wires a readiness probe will not // learn about a language mismatch until that first call fails. A host // that wants a real boot-time guarantee MUST call the exported @@ -282,67 +455,103 @@ export function createKnowledgePlane( engineConfig.ftsLanguage, ); - const plane: KnowledgePlane = { - async search(params) { - try { - await ensureVerified(); - const result = await hybridSearch(deps, { - tenantId: params.tenantId, - principalId: params.principalId, - query: params.query, - k: params.k, - kinds: params.kinds, - entityIds: params.entityIds, - }); + /** + * Hybrid retrieval + block-list post-filter. Shared by find() and ask(). + * Returns the full HybridSearchResult so ask can synthesize from hits. + */ + async function retrieve(params: { + tenantId: string; + principalId: string; + query: string; + k?: number; + kinds?: string[]; + entityIds?: string[]; + }): Promise { + try { + await ensureVerified(); + const result = await hybridSearch(deps, { + tenantId: params.tenantId, + principalId: params.principalId, + query: params.query, + ...(params.k !== undefined ? { k: params.k } : {}), + ...(params.kinds !== undefined ? { kinds: params.kinds } : {}), + ...(params.entityIds !== undefined + ? { entityIds: params.entityIds } + : {}), + }); - // Block-list post-filter: docs may store acl_block as a list of - // principal ids. Engine visibility does not model block lists yet. - // Shared with timeline via readBlockList / blockedDocumentIds. - if (result.hits.length === 0) return result; - const docIds = Array.from( - new Set(result.hits.map((h) => h.document_id)), - ); - const rows = await sql< - { id: string; attributes: Record | null }[] - >` + // Block-list post-filter: docs may store acl_block as a list of + // principal ids. Engine visibility does not model block lists yet. + // Shared with recent via readBlockList / blockedDocumentIds. + if (result.hits.length === 0) return result; + const docIds = Array.from( + new Set(result.hits.map((h) => h.document_id)), + ); + const rows = await sql< + { id: string; attributes: Record | null }[] + >` SELECT id, attributes FROM knowledge_document WHERE id = ANY(${docIds}::text[]) `; - const { blocked, unreadable } = blockedDocumentIds( - docIds, - rows, - params.principalId, + const { blocked, unreadable } = blockedDocumentIds( + docIds, + rows, + params.principalId, + ); + if (unreadable.length > 0) { + // Cap the sample so a large withhold batch cannot flood logs; count + // is always present so the full size is still auditable. + const sampleLimit = 20; + const documentIds = unreadable.slice(0, sampleLimit); + const more = + unreadable.length > sampleLimit + ? ` (+${unreadable.length - sampleLimit} more)` + : ""; + log.warn( + `find: ${unreadable.length} document(s) had an unreadable acl_block or missing row; withholding: ${documentIds.join(", ")}${more}`, + { count: unreadable.length, documentIds }, ); - if (unreadable.length > 0) { - // Cap the sample so a large withhold batch cannot flood logs; count - // is always present so the full size is still auditable. - const sampleLimit = 20; - const documentIds = unreadable.slice(0, sampleLimit); - const more = - unreadable.length > sampleLimit - ? ` (+${unreadable.length - sampleLimit} more)` - : ""; - log.warn( - `search: ${unreadable.length} document(s) had an unreadable acl_block or missing row; withholding: ${documentIds.join(", ")}${more}`, - { count: unreadable.length, documentIds }, - ); - } - if (blocked.size === 0) return result; - const hits = result.hits.filter((h) => !blocked.has(h.document_id)); - // result.evidence is already "none" only when there were no hits, so - // a post-filter that empties the list is the only way to reach "none". + } + if (blocked.size === 0) return result; + const hits = result.hits.filter((h) => !blocked.has(h.document_id)); + // result.evidence is already "none" only when there were no hits, so + // a post-filter that empties the list is the only way to reach "none". + return { + ...result, + hits, + evidence: hits.length === 0 ? "none" : result.evidence, + }; + } catch (err) { + if (err instanceof KnowledgeSearchInputError) { + throw new KnowledgeError(400, err.message); + } + throw err; + } + } + + const plane: KnowledgePlane = { + async find(params) { + const limit = resolveFindLimit(params.limit); + const result = await retrieve({ + tenantId: params.tenantId, + principalId: params.principalId, + query: params.query, + ...(limit !== undefined ? { k: limit } : {}), + ...(params.kinds !== undefined ? { kinds: params.kinds } : {}), + ...(params.entityIds !== undefined + ? { entityIds: params.entityIds } + : {}), + }); + const items = hitsToFindItems(result.hits); + if (params.includeEvidence) { return { - ...result, - hits, - evidence: hits.length === 0 ? "none" : result.evidence, + items, + evidence: result.evidence, + ...(result.degraded ? { degraded: result.degraded } : {}), }; - } catch (err) { - if (err instanceof KnowledgeSearchInputError) { - throw new KnowledgeError(400, err.message); - } - throw err; } + return { items }; }, async ask(params) { @@ -350,8 +559,9 @@ export function createKnowledgePlane( // HTTP surface's `requireGrant("knowledge", ...)` route guard, so the // check has to live here — AUTH.md is explicit that the capability and // data layers are independent and BOTH must allow. Per-document - // visibility (enforced inside `search`) is not a substitute for "may + // visibility (enforced inside `find`) is not a substitute for "may // this principal search at all". +// HTTP routes still guard with action "search"; ask matches that. if (!grants) { throw new KnowledgeError( 501, @@ -395,56 +605,106 @@ export function createKnowledgePlane( ); } - // Search AS the asking principal — the per-document ACL boundary, - // including the block-list post-filter above. - const result = await plane.search({ + // Find AS the asking principal — the per-document ACL boundary, + // including the block-list post-filter. includeEvidence so synthesis + // can report evidence; goes through plane.find so tests can stub it. + const findResult = await plane.find({ tenantId: params.tenantId, principalId: params.principalId, query: params.query, - ...(params.k !== undefined ? { k: params.k } : {}), + includeEvidence: true, + ...(params.limit !== undefined ? { limit: params.limit } : {}), }); - return synthesizeAnswer(params.query, result, options.generate); + return synthesizeAnswer( + params.query, + { + hits: findItemsToHits(findResult.items), + evidence: findResult.evidence ?? "none", + }, + options.generate, + ); }, - async capture(params) { + async add(params) { await ensureVerified(); + + const hasContent = params.content !== undefined; + const hasFile = params.file !== undefined; + if (hasContent === hasFile) { + throw new KnowledgeError( + 400, + "provide exactly one of content or file", + ); + } + + let title: string; + let text: string; + if (params.content) { + title = params.content.title; + text = params.content.text; + } else { + const file = params.file!; + if (!options.textExtractor) { + throw new KnowledgeError( + 400, + "file requires a textExtractor on the knowledge plane", + ); + } + const extracted = await options.textExtractor.extract({ + bytes: file.bytes, + ...(file.mimeType !== undefined ? { mimeType: file.mimeType } : {}), + ...(file.filename !== undefined ? { filename: file.filename } : {}), + }); + text = extracted.text; + title = + file.title ?? + extracted.title ?? + file.filename ?? + "untitled"; + } + + const { visibility, blockPrincipalIds } = + resolveShareAndVisibility(params); + const adapter = params.adapter ?? "mcp"; const externalRef = params.externalRef ?? `knowledge:${params.tenantId}:${crypto.randomUUID()}`; - const visibility = params.visibility ?? { mode: "tenant" as const }; const attributes: Record = { ...(params.attributes ?? {}), }; - if (params.blockPrincipalIds && params.blockPrincipalIds.length > 0) { - attributes["acl_block"] = JSON.stringify(params.blockPrincipalIds); + if (blockPrincipalIds && blockPrincipalIds.length > 0) { + attributes["acl_block"] = JSON.stringify(blockPrincipalIds); } - await captureDocument(deps, { + const captureResult = await captureDocument(deps, { tenantId: params.tenantId, adapter, occurredAt: new Date().toISOString(), document: { kind: params.kind ?? "note", - title: params.title, + title, externalRef, visibility, entityHints: [], - chunks: [{ ordinal: 0, text: params.text }], + chunks: [{ ordinal: 0, text }], actor: { kind: "human", principalId: params.principalId }, contentHash: "", // recomputed canonically in adapt-and-plan ...(Object.keys(attributes).length > 0 ? { attributes } : {}), }, }); + // Both captured and noop return documentId — always surface it. + return { documentId: captureResult.documentId }; }, - async timeline(params) { + async recent(params) { + const limit = resolveRecentLimit(params.limit); return listTimelineEvents({ db, tenantId: params.tenantId, principalId: params.principalId, - ...(params.limit !== undefined ? { limit: params.limit } : {}), + ...(limit !== undefined ? { limit } : {}), }); }, diff --git a/src/routes/capture.ts b/src/routes/capture.ts index 72d40a1..8737119 100644 --- a/src/routes/capture.ts +++ b/src/routes/capture.ts @@ -14,7 +14,10 @@ const CaptureRequest = type({ "acl?": "unknown", }); -const CaptureResponse = type({ status: "'captured'" }); +const CaptureResponse = type({ + status: "'captured'", + documentId: "string", +}); export function mountCaptureRoute(app: Hono, deps: RouteDeps): void { app.post( @@ -45,15 +48,14 @@ export function mountCaptureRoute(app: Hono, deps: RouteDeps): void { if (!parsed.ok) return c.json({ error: parsed.error }, 400); try { - await deps.knowledge.capture({ - title, - text, + const { documentId } = await deps.knowledge.add({ + content: { title, text }, tenantId: scopeId, principalId: subjectId, visibility: parsed.visibility, blockPrincipalIds: parsed.block, }); - return c.json({ status: "captured" }); + return c.json({ status: "captured", documentId }); } catch (err) { const errMessage = formatCaughtError(err); log.error(`knowledge capture failed: ${errMessage}`, { error: errMessage }); diff --git a/src/routes/routes.test.ts b/src/routes/routes.test.ts index 5fdd350..ca232b4 100644 --- a/src/routes/routes.test.ts +++ b/src/routes/routes.test.ts @@ -29,8 +29,8 @@ const TENANT = "t1"; const SECRET_TITLE = "Q3 layoffs — draft list"; const PUBLIC_TITLE = "team standup notes"; -// A knowledge plane stub that records captures and returns fixed results. -// Timeline applies a simple ACL model so route tests can prove the route +// A knowledge plane stub that records adds and returns fixed results. +// Recent applies a simple ACL model so route tests can prove the route // never invents titles and always scopes by the caller's principal. function stubPlane(opts?: { timelineCatalog?: Array< @@ -40,23 +40,27 @@ function stubPlane(opts?: { const captured: { title: string; tenantId: string; principalId: string }[] = []; const searched: Array< - Pick[0], "kinds" | "entityIds" | "k"> + Pick< + Parameters[0], + "kinds" | "entityIds" | "limit" + > > = []; const catalog = opts?.timelineCatalog ?? []; const plane: KnowledgePlane = { - search: async (p) => { - searched.push({ kinds: p.kinds, entityIds: p.entityIds, k: p.k }); - return { hits: [], evidence: "none" }; + find: async (p) => { + searched.push({ kinds: p.kinds, entityIds: p.entityIds, limit: p.limit }); + return { items: [], evidence: "none" }; }, ask: async () => ({ text: "", citations: [], evidence: "none" }), - capture: async (p) => { + add: async (p) => { captured.push({ - title: p.title, + title: p.content?.title ?? "", tenantId: p.tenantId, principalId: p.principalId, }); + return { documentId: "doc-stub" }; }, - timeline: async (p) => { + recent: async (p) => { return catalog .filter( (e) => @@ -177,6 +181,12 @@ describe("knowledge HTTP routes", () => { jsonPost({ title: "t", text: "body" }), ); expect(res.status).toBe(200); + const body = (await res.json()) as { + status: string; + documentId: string; + }; + expect(body.status).toBe("captured"); + expect(body.documentId).toBe("doc-stub"); expect(captured).toEqual([ { title: "t", tenantId: TENANT, principalId: PRINCIPAL }, ]); @@ -208,7 +218,11 @@ describe("knowledge HTTP routes", () => { jsonPost({ query: "hello" }), ); expect(res.status).toBe(200); - const body = (await res.json()) as { evidence: string }; + const body = (await res.json()) as { + items: unknown[]; + evidence: string; + }; + expect(body.items).toEqual([]); expect(body.evidence).toBe("none"); }); @@ -242,7 +256,11 @@ describe("knowledge HTTP routes", () => { ); expect(res.status).toBe(200); expect(searched).toEqual([ - { kinds: ["artifact", "task"], entityIds: ["e1", "e2"], k: undefined }, + { + kinds: ["artifact", "task"], + entityIds: ["e1", "e2"], + limit: undefined, + }, ]); }); @@ -254,7 +272,7 @@ describe("knowledge HTTP routes", () => { ); expect(res.status).toBe(200); expect(searched).toEqual([ - { kinds: undefined, entityIds: undefined, k: undefined }, + { kinds: undefined, entityIds: undefined, limit: undefined }, ]); }); @@ -276,7 +294,9 @@ describe("knowledge HTTP routes", () => { jsonPost({ query: "hello", kinds: [], entity_ids: [] }), ); expect(res.status).toBe(200); - expect(searched).toEqual([{ kinds: [], entityIds: [], k: undefined }]); + expect(searched).toEqual([ + { kinds: [], entityIds: [], limit: undefined }, + ]); }); test("timeline requires the search grant", async () => { diff --git a/src/routes/search.ts b/src/routes/search.ts index f9b679e..e9ebe2d 100644 --- a/src/routes/search.ts +++ b/src/routes/search.ts @@ -5,7 +5,6 @@ import { type } from "arktype"; import { formatCaughtError, log } from "../log.ts"; import { KnowledgeError } from "../knowledge.ts"; -import { SearchResponseSchema } from "../core/schemas/search.ts"; import type { RouteDeps } from "./deps.ts"; import { caller, grantGuard, requirePrincipal } from "./deps.ts"; @@ -23,6 +22,21 @@ const SearchRequest = type({ "entity_ids?": "string[]", }); +// Green FindResult shape. includeEvidence is always true on HTTP so the wire +// keeps reporting evidence for existing clients. +const FindResponse = type({ + items: type({ + documentId: "string", + title: "string", + snippet: "string", + score: "number", + kind: "string", + citation: "unknown", + }).array(), + "evidence?": "'strong'|'weak'|'none'", + "degraded?": "string[]", +}); + export function mountSearchRoute(app: Hono, deps: RouteDeps): void { app.post( "/api/knowledge/search", @@ -35,9 +49,9 @@ export function mountSearchRoute(app: Hono, deps: RouteDeps): void { "requested kind/entity.", responses: { 200: { - description: "Ranked hits with evidence", + description: "Ranked items with evidence", content: { - "application/json": { schema: resolver(SearchResponseSchema) }, + "application/json": { schema: resolver(FindResponse) }, }, }, 400: { description: "Invalid query" }, @@ -52,11 +66,14 @@ export function mountSearchRoute(app: Hono, deps: RouteDeps): void { const { query, k, kinds, entity_ids } = c.req.valid("json"); const { scopeId, subjectId } = caller(c); try { - const result = await deps.knowledge.search({ +// Body still accepts k — map to green limit. Always includeEvidence so + // the wire keeps the evidence field clients already rely on. + const result = await deps.knowledge.find({ query, tenantId: scopeId, principalId: subjectId, - ...(k !== undefined ? { k } : {}), + includeEvidence: true, + ...(k !== undefined ? { limit: k } : {}), ...(kinds !== undefined ? { kinds } : {}), ...(entity_ids !== undefined ? { entityIds: entity_ids } : {}), }); diff --git a/src/routes/timeline.ts b/src/routes/timeline.ts index 567892c..54c16ef 100644 --- a/src/routes/timeline.ts +++ b/src/routes/timeline.ts @@ -40,7 +40,7 @@ export function mountTimelineRoute(app: Hono, deps: RouteDeps): void async (c) => { const { scopeId, subjectId } = caller(c); try { - const events = await deps.knowledge.timeline({ +const events = await deps.knowledge.recent({ tenantId: scopeId, principalId: subjectId, });