diff --git a/.gitignore b/.gitignore index 6f32458..98a4ba4 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,7 @@ coverage/ .hub-data/ .agent-state/ tmp/ + +# Dispatch orchestration scratch (local only) +dispatch/ + diff --git a/AGENTS.md b/AGENTS.md index 295c0b9..e941598 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,7 @@ # Agent guide — @corbits/knowledge-engine -A library, not a service. `src/` is the whole product: a knowledge capture + -search SDK that **mounts onto a host Interchange app**. There is no server, +A library, not a service. `src/` is the whole product: a knowledge add / find / +ask / recent SDK that **mounts onto a host Interchange app**. There is no server, port, or process entrypoint here, and there never should be. ## Commands @@ -20,10 +20,12 @@ CI runs `typecheck` + `test` — both must pass before any push. - `src/index.ts` — public surface: `mountKnowledgeEngine`, `mountKnowledgeRoutes`, `createKnowledgePlane` - `src/mount-config.ts` / `src/config.ts` — mount config + engine config -- `src/routes/` — Hono routes (capture, search, timeline) -- `src/services/` — capture / search / transform logic -- `src/core/` — embed/rerank clients, arktype schemas -- `src/db/` + `migrations/` — Drizzle schema + SQL migrations (pgvector) +- `src/routes/` — Hono routes (`add`, `find`, `ask`, `recent`) +- `src/services/` — capture / search / transform internals (not public verbs) +- `src/ports/` — `DocumentStore` / `SourceProvider` / `MemoryProvider` + fakes +- `src/core/` — embed/rerank clients, merge, arktype schemas +- `src/db/` + `migrations/` — Drizzle schema + SQL migrations (pgvector, `knowledge.*`) +- `packages/` — optional DocumentStore adapters (`knowledge-adapter-mem0`, `knowledge-adapter-supermemory`); pure fetch, no vendor SDKs in core. Linear tools live in sibling `@corbits/linear`. ## Non-negotiable invariants diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 8e87935..f974b81 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,6 +1,6 @@ # Knowledge Engine — Architecture -A knowledge capture + retrieval SDK that mounts onto an Interchange hub. The +A knowledge add / find / ask / recent SDK that mounts onto an Interchange hub. The host owns auth, tenancy, and the process; this library owns the knowledge / vector plane and the routes that read and write it. @@ -66,25 +66,29 @@ model. `mountKnowledgeEngine` adds, under the host app: -- `POST /api/knowledge/capture` — ingest content (raw + derive). -- `POST /api/knowledge/search` — hybrid retrieval: FTS + dense (pgvector) → RRF - fusion → cross-encoder rerank → bounded authority/recency boosts → MMR. -- `GET /api/knowledge/timeline` — recent captures for the caller's scope, - filtered with the same document ACL as search (visibility + block list). - -It also returns an in-process `KnowledgePlane` (`capture`, `search`, `ask`). -`ask()` is grant-checked in-process (callers bypass the HTTP `requireGrant` -guard), searches as the asking principal, grounds a prompt from hit `snippet` -text (search truncates snippets to ≤240 chars today — that is the MVP -grounding limit), and calls a host-injected `generate` function. The engine -owns no generation client; hosts wire `generate` to their inference layer. +- `POST /api/knowledge/add` — ingest a note (raw + derive). +- `POST /api/knowledge/find` — hybrid retrieval: FTS + dense (pgvector) → RRF + fusion → cross-encoder rerank → bounded authority/recency boosts → MMR; + optional live `SourceProvider` merge (fail-soft). +- `POST /api/knowledge/ask` — grant-checked as `knowledge:find`; retrieves as + the principal, grounds a prompt from hit snippets, calls host-injected + `generate`. Optional memory recall when `includeMemory` is true. +- `GET /api/knowledge/recent` — recent documents for the caller's scope, + filtered with the same document ACL as local find (visibility + block list). + +It also returns an in-process `KnowledgePlane` (`add`, `find`, `ask`, +`recent`, optional `remember` / `recall`). `ask()` is grant-checked in-process +(callers bypass the HTTP `requireGrant` guard). The engine owns no generation +client; hosts wire `generate` to their inference layer. MCP is not part of this package — mount `@corbitsdev/hono-openapi-mcp` to expose these routes as MCP tools. External ingestion (Linear, GitHub, …) is not a route here — the host -authenticates the forwarder to Interchange and it calls the capture route, or -the host calls `knowledge.capture()` directly. +authenticates the forwarder to Interchange and calls `plane.add` / a +`SourceProvider` mapper, or mounts HTTP add after its own auth. + +Legacy paths `/capture`, `/search`, `/timeline` are not mounted (hard cutover). ## Provenance diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md index 87b0b94..2a43c3d 100644 --- a/IMPLEMENTATION.md +++ b/IMPLEMENTATION.md @@ -11,29 +11,33 @@ src/ index.ts # mountKnowledgeEngine / mountKnowledgeRoutes mount-config.ts # KnowledgeConfig + loadKnowledgeConfig() — the mount config config.ts # EngineConfig — the core vector-plane config (db + embed + rerank) - knowledge.ts # createKnowledgePlane — capture/search/timeline against pgvector + knowledge.ts # createKnowledgePlane — add/find/ask/recent against store or pgvector acl.ts # parseAcl + shared acl_block read-path helpers log.ts # getLogger(["knowledge-engine"]) from @intx/log migrations.ts # runKnowledgeMigrations(url) + ports/ # DocumentStore / SourceProvider / MemoryProvider + fakes routes/ # the mounted routes mount.ts # mountKnowledgeRoutes (HTTP) deps.ts # RouteDeps, caller(c) (context identity), grantGuard - capture.ts, search.ts, timeline.ts + add.ts, find.ts, ask.ts, recent.ts db/ - schema.ts # Drizzle table defs for every fixed-shape table + schema.ts # Drizzle table defs (knowledge.* schema) client.ts # createDb(config) -> { db (drizzle), sql (raw postgres-js) } services/ capture.ts # captureDocument, deriveFromRawCapture — the write path search.ts # hybridSearch and every retrieval-candidate query timeline.ts # listTimelineEvents — durable recent docs + ACL filter transform.ts # transform_config CRUD + runTransform (replay) - core/ # framework-agnostic, mostly pure (chunking, embed/rerank - # clients, authority, hybrid fusion, MMR, schemas) + core/ # framework-agnostic (chunking, embed/rerank, merge, schemas) +packages/ + knowledge-adapter-mem0/ # DocumentStore backend (Mem0) + knowledge-adapter-supermemory/ # DocumentStore backend (Supermemory) migrations/ # pgvector schema, applied in filename order by scripts/db-setup.ts scripts/db-setup.ts # idempotent migration runner, tracked in `_migrations` compose.yml # pgvector + Ollama + reranker for local dev ``` + The SDK has no server and no process entrypoint. `mountKnowledgeEngine` takes the host's `Hono` app plus `{ config, grants? }` and mounts the routes; each reads identity from the context (`caller(c)`) and guards via @@ -126,7 +130,7 @@ columns (`authority`, `actor_count`, `has_social_signal`, `source_class`) are a **snapshot computed once at capture time** (`computeAuthority`, never recomputed retroactively). `raw_capture_id` points at the immutable source row this version was derived from. `generation` (added by migration 0009, -default `'live'`) is the replay-generation tag: the normal `/capture` path always writes +default `'live'`) is the replay-generation tag: the normal add path always writes `'live'`; a replay (`runTransform`) writes its own `transform_run.id` instead, so a replayed corpus's versions never collide with, or even become visible alongside, the live ones unless a caller explicitly searches that generation. @@ -245,7 +249,7 @@ string-interpolated into raw SQL — this is the only place in the codebase a computed identifier is spliced into DDL/DML. ### `raw_capture` -The immutable, append-only substrate (the raw-capture layer). Stores the exact `/capture` +The immutable, append-only substrate (the raw-capture layer). Stores the exact add/ingest request payload (`adapter`, `occurred_at`, `document`) as JSON in `raw_text` (there's also a `raw_bytes bytea` column for non-textual payloads, currently unused by any write path — everything captured today is JSON). Deduped on @@ -320,7 +324,7 @@ returns the run summary either way. this repo; chunks left unembedded simply never populate the dense channel for the query — they're still found by lexical/FTS). Any of these failure modes sets `degraded: true` on the `CaptureResult`, surfaced by - `POST /api/knowledge/capture` as a `degraded` field in its response — the capture + `POST /api/knowledge/add` as a `degraded` field in its response — the add still succeeded (chunks are durable and lexically searchable), only the dense/vector channel for those chunks is incomplete. @@ -493,7 +497,7 @@ timeline maps different columns: |---|---| | `at` | `knowledge_document.last_seen_at` (ISO) — re-captures rise in the feed | | `title` | `knowledge_document.title` | -| `source` | `knowledge_document.adapter` (HTTP capture defaults to `"mcp"`, not `"api"`) | +| `source` | `knowledge_document.adapter` (HTTP add defaults to `"http"`, not `"api"`) | | `tenantId` | `knowledge_document.tenant_id` | | `principalId` | `knowledge_version.created_by_principal_id` of the active live version (empty string when null) — the capturing actor stored on the version, not the request principal of a later timeline read | diff --git a/PRODUCT.md b/PRODUCT.md index a24643b..e94d9b2 100644 --- a/PRODUCT.md +++ b/PRODUCT.md @@ -1,135 +1,155 @@ # Corbits Knowledge Engine — Product shape -One product a team can run **without Workbench**: company brain for Claude Code, -Codex, and any other client. Workbench is a client, not the owner of ingestion -or auth. +A **mountable knowledge plane** for Interchange hubs: durable documents, hybrid +search, grounded ask, and optional live sources / personal memory. Workbench and +coding agents are clients — not owners of ingestion or auth. ## Shape (locked) -**`src/` is the `@corbits/knowledge-engine` SDK. Interchange is the hub — the -SDK never creates one; it mounts onto yours. Nothing else ships in this repo.** - -1. **Public surface**: `mountKnowledgeEngine(app, opts)` drops the knowledge - plane + routes onto an Interchange `createApp`; - `createKnowledgePlane(config)` builds the same plane without mounting HTTP - (CLI seeders, batch ingesters, tests); - `runKnowledgeMigrations(url)` applies the pgvector schema; - `loadKnowledgeConfig()` / `KnowledgeConfig` is the mount config. -2. **The SDK authenticates nothing.** Identity is the request principal, read - off the Interchange context (`c.get("principal")`). Authorization uses the - host's grant store via Interchange's `authorize` / `createRequireGrant`. No API keys, no - sessions, no OAuth, no membership resolution — Interchange did all of that - before the request reached a mounted route. -3. **Knowledge data plane runs in-process** against the host's knowledge/vector - Postgres. No second server, no HTTP hop. +**`src/` is the `@corbits/knowledge-engine` SDK.** Interchange is the hub — the +SDK never creates one; it mounts onto yours. -### What is not in scope +| Surface | Role | +| --- | --- | +| `mountKnowledgeEngine(app, opts)` | Plane + HTTP on an Interchange `createApp` | +| `createKnowledgePlane(config, grants?, options?)` | Same plane without HTTP | +| `runKnowledgeMigrations(url)` | Apply pgvector schema under Postgres `knowledge` | +| `loadKnowledgeConfig()` | Mount config from env | -- No auth, no API keys, no OAuth, no webhook, no SPA, no standalone server. - Those belong to the host (a reference deployment lives in - `corbitsdev/examples`, not here). -- Workbench is **not** required. Workbench / Claude Code / Codex are **clients**. +### Green public plane (only these verbs) -**One Postgres for the SDK:** the knowledge / vector store (`pgvector`) — for -capture, hybrid search, raw corpus, replay. The host's control-plane DB (auth, -tenants, principals, grants, sessions) is entirely the host's concern. +| Method | Meaning | +| --- | --- | +| `add` | Capture a document (`content` **xor** `file` + TextExtractor) | +| `find` | Hybrid retrieval (+ optional live sources) | +| `ask` | Grounded answer from find (+ optional memory) | +| `recent` | Recent documents for the principal | -``` -Claude Code / Codex / Workbench (clients) - │ authenticated by Interchange (session | API key | MCP OAuth) - │ body: content (+ optional acl) — never tenant/principal - ▼ -┌──────────────────────────────────────────┐ -│ Host Interchange createApp │ -│ resolves principal + tenant + grants │ -│ + mountKnowledgeEngine(app, opts) │ -│ or createKnowledgePlane(config) │ -│ reads c.get("principal") from context │ -│ guards via host requireGrant(...) │ -│ │ in-process │ -│ ▼ │ -│ Knowledge plane (capture / search) │ -│ store under scope_id + document ACL │ -│ search: scope first, then ACL match │ -│ │ -│ Knowledge / vector Postgres (pgvector) │ -└──────────────────────────────────────────┘ -``` +Hard cutover: there is no `capture` / `search` / `timeline` export. HTTP paths +and grants match the verbs: `POST /api/knowledge/add|find|ask`, +`GET /api/knowledge/recent`; grants `knowledge:add` and `knowledge:find` +(ask/recent share `find`). -## Identity and ACL +Identity on the plane is always **`principalId` + `tenantId`** (never +`scopeId` / `subjectId`). HTTP routes never take body identity — they read +`c.get("principal")` from Interchange context. -**The SDK does not authenticate.** Interchange authenticates the caller -(session, API key, or MCP OAuth) and puts `principal` + `tenant` on the request -context. Each mounted route reads identity from there: -`scopeId = principal.tenantId`, `subjectId = principal.id`. Clients never send -`tenant_id`/`principal_id` — the routes ignore body identity entirely. +### Ports (pluggable) -Access is gated by the host's grant system: pass `grants` (the same -`{ grantStore, conditionRegistry }` you give `createApp`) to -`mountKnowledgeEngine`. HTTP routes run `requireGrant("knowledge", )` -(`capture` for capture, `search` for search/timeline). -`createKnowledgePlane` builds the same plane without routes — callers acting -for a user must check the capability themselves (see README). +| Port | Purpose | +| --- | --- | +| `DocumentStore` | **The** durable backend for add/find/recent (default: engine pgvector, wrapped as a DocumentStore). Replace with Mem0, Supermemory, or fakes — no Postgres required when overridden. The plane is always store-backed; there is no second engine-only path. | +| `SourceProvider` | Optional **tools-shaped** live search (`searchLive`); merge is fail-soft. Not a store replacement. | +| `MemoryProvider` | Optional ask side-channel only (`includeMemory`); **not** how you swap backends. | -### On the wire +Mount options accept `documentStore`, `sources[]`, `memory`, plus in-package +**fakes** so a host can mount with fakes only and exercise add/find/ask/recent +without Postgres. Hosts that want Mem0 or Supermemory as the sole durable store +pass that adapter as `documentStore` and omit `KnowledgeConfig`. -```http -POST /api/knowledge/capture { "title", "text", "acl?" } -POST /api/knowledge/search { "query", "k?", "kinds?", "entity_ids?" } -GET /api/knowledge/timeline -``` +**MergeLocalLiveV1** merges local DocumentStore + live SourceProviders: fail-soft +per provider (timeout/error → degrade flags, never fail the request), dedupe by +`adapter:externalRef`, optional `sources` filter (`local` + provider ids). + +**Memory side-channel:** `includeMemory` on `ask` defaults **false**. When true +and a `MemoryProvider` is mounted, recall injects uncited personal context; +failures degrade with `memory_unavailable` (docs-only). This is unrelated to +using Mem0/Supermemory as the DocumentStore. Writes via `plane.remember` are +host-owned — ask never auto-writes. + +### Adapter packages (same monorepo tree) + +| Package | Role | +| --- | --- | +| `@corbits/knowledge-adapter-mem0` | **DocumentStore** via Mem0 Platform HTTP; tenant key `mapUser` length-prefixed | +| `@corbits/knowledge-adapter-supermemory` | **DocumentStore** via Supermemory HTTP; tenant key `containerTag` length-prefixed | + +Linear tools live in a **sibling repo** ([`@corbits/linear`](https://github.com/corbitsdev/corbits-linear)) — same tools shape as Granola, not a DocumentStore. -Requests are authenticated upstream by Interchange (however the host chose); -the SDK routes assume a resolved principal on the context. +Core never imports vendor SDKs. Adapters are pure-fetch; tenant-safe keys only. +`MemoryProvider` factories in the mem0/supermemory packages are back-compat only. -`kinds`/`entity_ids` narrow both the lexical and dense/semantic legs of -search before results are fused, so every hit matches the requested -kind/entity. An empty array on either field is equivalent to omitting it (no -filter), not "match nothing". +**Vendor store honesty:** Mem0/Supermemory adapters isolate by **principal +bucket** (one Mem0 `user_id` / Supermemory `containerTag` per tenant+principal). +They do **not** implement the multi-principal / tenant visibility ladder or +block lists — those need the default pgvector store (or a store that enforces +them). `recent` is empty on both adapters. Never mount them as +`options.memory`. + +### What is not in scope -### Document ACL (who may surface on search/timeline) — set at capture +- No auth, API keys, OAuth, webhooks, SPA, or standalone server in core. +- No dual ACL / aspirational `source_acl` writes as a security boundary. +- Workbench is a client, not required. +- Linear / Granola / MCP tools are **not** DocumentStore replacements. -Optional on capture; default is scope-wide (the company brain), or -private-to-subject. +**Default durable store:** local Postgres via `KNOWLEDGE_DATABASE_URL` only (no +`DATABASE_URL` fallback), tables under the **`knowledge`** schema. When +`documentStore` is injected, Postgres is not opened. Cross-refs (`tenant_id`, +`principal_id`) on the default store are plain `text` — no FKs into the host +control plane. -```ts -acl?: { - mode: "scope" | "tenant" | "private" | "allowlist" - allow?: string[] | { subjects?: string[] } // subjects only for now - block?: string[] | { subjects?: string[] } -} +``` +Claude Code / Codex / Workbench (clients) + │ authenticated by Interchange + ▼ +┌──────────────────────────────────────────────┐ +│ Host Interchange createApp │ +│ + mountKnowledgeEngine(app, opts) │ +│ grants: knowledge:add | knowledge:find │ +│ documentStore: pgvector | Mem0 | SM | │ +│ fake │ +│ optional: sources, memory, │ +│ textExtractor │ +│ │ in-process │ +│ ▼ │ +│ Knowledge plane: add / find / ask / recent │ +│ → DocumentStore (sole durable backend) │ +└──────────────────────────────────────────────┘ ``` -Search and timeline both filter by scope first, then apply the document ACL -against the caller's subject. Block wins over allow. (Group/grant-based ACLs -are rejected until membership lands.) Timeline reads durable document state — -not a process-local capture ring — so ACL changes and restarts do not leak -titles. Each timeline event's `source` is the document adapter (HTTP capture -defaults to `"mcp"`; it is no longer hardcoded `"api"`), and `principalId` is -the capturing actor on the active version (`created_by_principal_id`), not the -caller of the timeline request. +## Identity and ACL ladder (honest) -## Ingestion +1. **Capability** — host grant store: may this principal `knowledge:add` or + `knowledge:find` at all? +2. **Document visibility** — modes `private` | `principals` | `tenant` + (optional block list). Self-contained on the document row. +3. **Share sugars on add** — map to existing visibility; no new ACL system. -Ingestion is **capture** via the HTTP capture route, under the caller's context identity. (Expose it as an MCP tool with `@corbitsdev/hono-openapi-mcp`.) +There is **no** dual grant path and **no** second secret ACL. If a connector +cannot prove a principal set, it must not write `tenant` visibility. -External events (Linear, GitHub, …) come in the same way: the host authenticates -the forwarder to Interchange (e.g. an API key issued to the integration) and it -calls `POST /api/knowledge/capture`, or the host calls `knowledge.capture()` -directly from its own inbound handler. Vendor-payload mapping lives at the edge, -not in the SDK. +### On the wire +HTTP bodies are a thin subset of the in-process plane (identity always comes +from the host principal context, never the body): -## Clients +```http +POST /api/knowledge/add { "title", "text", "acl"? } +POST /api/knowledge/find { "query", "limit?", "kinds?", "entity_ids?", "sources?", "includeEvidence?" } +POST /api/knowledge/ask { "query", "limit?", "sources?", "includeMemory?" } +GET /api/knowledge/recent ?limit= +``` -| Client | How it talks to it | -| --- | --- | -| Claude Code / Codex | MCP + OAuth (or API key) on the host | -| Workbench | Generic tool client (not owner of ingest) | -| Humans | Host app's UI (sign-in, timeline) | +`kinds` / `entity_ids` on find narrow both lexical and dense channels before +fusion (unset or `[]` = no filter). + +Plane-only shapes (`content`/`file` XOR, `share` sugars, full `visibility`) +are available via `createKnowledgePlane` / `plane.add`. HTTP `acl` maps to the +document visibility ladder; `share` is plane sugar only. + +### Live sources and memory (trust) + +- **Local documents** are the durable ACL plane (visibility + block). Engine + path enforces this; a host-supplied `DocumentStore` **owns** ACL for that + mount — the engine does not re-filter store results. +- **Live `SourceProvider` hits** merge into find/ask without document-row ACL. + Auth is the host token / connector scope, not Interchange principal + visibility. Treat live as enrichment; fail-soft on timeout/error. +- **Memory** is opt-in recall (`includeMemory`, default false). Adapters must + key by injective tenant+principal encodings; ask never auto-writes memory. -## Non-goals (this cut) +## Out of scope forever here -- Workbench owning ingestion or tenancy for the company brain. -- Client-supplied `tenant_id` / `principal_id` on the routes. -- Shipping without Interchange as control plane. +Auth, OAuth for Linear, Mem0/Supermemory account management, embedding models +in-process, and any standalone process entrypoint. diff --git a/README.md b/README.md index 676cc0d..0489713 100644 --- a/README.md +++ b/README.md @@ -1,297 +1,150 @@ # @corbits/knowledge-engine -A knowledge capture + search engine you **mount** onto an [Interchange](https://github.com/corbitsdev) hub. +Mountable knowledge plane for [Interchange](https://github.com/corbitsdev) hubs: +**add** documents, **find** with hybrid search, **ask** grounded answers, **recent** +timeline — with optional live sources and personal memory. -`mountKnowledgeEngine(app, opts)` adds hybrid semantic + keyword search, capture -(with per-document ACLs), and a grant-checked in-process `ask()` to your -Interchange `createApp`. - -MCP is a separate concern: mount `@corbitsdev/hono-openapi-mcp` to expose these -(or any documented) routes as MCP tools — no MCP code lives in this package. - -**It authenticates nothing.** Identity is the request principal, read straight -off the Interchange context (`c.get("principal")`); authorization goes through -the host's own grant store (`@intx/authz`). No API keys, no sessions, no OAuth -live here — Interchange already did all of that before the request reaches a -mounted route. - -**The engine never embeds in-process.** Every capture and search call goes out -to an embedding endpoint you configure — any OpenAI-compatible, TEI, or -Ollama-style API. +**Authenticates nothing.** Identity is `c.get("principal")` on HTTP; in-process +callers pass `principalId` + `tenantId`. Authorization is the host grant store +(`knowledge:add` / `knowledge:find`). Never embeds in-process — embedding and +rerank are outbound HTTP to configured endpoints. Requires Bun 1.2+. -## What you get - -- **Capture / search / timeline** HTTP routes, each guarded with - `requireGrant("knowledge", )`. Timeline titles use the same document - ACL as search (visibility modes + fail-closed block lists). -- **Grant-checked `ask()`** on the plane: retrieves as the principal, grounds a - host-supplied `generate` callback, returns citations. Safe to call outside - HTTP because the capability check lives inside the method. -- **Out-of-band plane** via `createKnowledgePlane` for CLI seeders, batch - ingesters, and tests — no Hono app required. -- **Configurable lexical language** (`FTS_LANGUAGE`, default `english`) baked - into the generated tsvector at migration time and verified at query time. -- **Multi-model embeddings** with per-model tables; models above 2000 dims use - halfvec expression indexes (up to 4000) so dense search can still hit an index. -- **Optional cross-encoder rerank** (TEI, Cohere v2, or Voyage). Retrieval - degrades to fusion-only when unset. Hosts can poll degrade-metrics snapshots - to surface silent degradation without the engine owning a `/metrics` port. - ## Install ```bash bun add @corbits/knowledge-engine ``` -## Mount it +## Mount (green path) ```ts import { mountKnowledgeEngine } from "@corbits/knowledge-engine"; import { loadKnowledgeConfig } from "@corbits/knowledge-engine/config"; -// `app` is your Interchange createApp (Hono). Pass the same grant -// store + condition registry you give createApp/createRequireGrant. mountKnowledgeEngine(app, { - config: loadKnowledgeConfig(), // or build the object yourself + config: loadKnowledgeConfig(), grants: { grantStore, conditionRegistry }, + // optional ports: + // documentStore, sources, memory, textExtractor, generate }); ``` -That mounts `POST /api/knowledge/add`, `POST /api/knowledge/find`, -`POST /api/knowledge/ask`, and `GET /api/knowledge/recent`, each guarded with -`requireGrant("knowledge", )` (`add` or `find`). Clients never send -tenant or principal — identity is the context principal. - -### The host must resolve tenant + principal for `/api/knowledge/*` - -These routes read `c.get("principal")`, but they mount at `/api/knowledge/*` — -**outside** `/api/tenants/:tenantId/*`, which is where Interchange's own -`createResolveTenant` middleware is scoped and where it reads the tenant from -the path param. Nothing populates the context for our prefix, so the host has to: - -```ts -// Mount BEFORE mountKnowledgeEngine — the grant guard runs first and needs a -// principal on the context. -app.use("/api/knowledge/*", async (c, next) => { - const user = c.get("user"); - if (!user) return c.json({ error: "unauthorized" }, 401); - - // Your choice how the tenant is selected — a header, a subdomain, or the - // user's only membership. There is no path param to read. - const tenantRow = await resolveTenantSomehow(c); - const principalRow = await db.query.principal.findFirst({ - where: and( - eq(principal.tenantId, tenantRow.id), - eq(principal.kind, "user"), - eq(principal.refId, user.id), - ), - }); - if (!principalRow) return c.json({ error: "not a member" }, 403); - if (principalRow.status !== "active") - return c.json({ error: "inactive" }, 403); - - c.set("tenant", tenantRow); - c.set("principal", principalRow); - await next(); -}); -``` - -Without it every knowledge route returns **401 `principal_required`**. +Routes (each grant-checked): -### Capturing and searching outside a request +| Method | Path | Grant | +| --- | --- | --- | +| POST | `/api/knowledge/add` | `knowledge:add` | +| POST | `/api/knowledge/find` | `knowledge:find` | +| POST | `/api/knowledge/ask` | `knowledge:find` | +| GET | `/api/knowledge/recent` | `knowledge:find` | -`mountKnowledgeEngine` returns the `KnowledgePlane` it built, and the plane takes -identity as data — so an in-process caller passes `{tenantId, principalId}` -explicitly rather than faking a request context: +Clients never send tenant/principal in the body. -```ts -const { knowledge } = mountKnowledgeEngine(app, { config, grants }); +### Host must set principal on `/api/knowledge/*` -await knowledge.search({ tenantId, principalId, query: "…", k: 6 }); -``` +These routes sit outside `/api/tenants/:tenantId/*`. Mount middleware **before** +`mountKnowledgeEngine` that sets `c.set("principal", …)` and `c.set("tenant", …)`. +Without it: **401 `principal_required`**. -For a CLI seeder, a batch ingester, or a test with no app at all, construct a -plane directly: +### Plane without HTTP ```ts import { createKnowledgePlane, - loadKnowledgeConfig, + createFakeDocumentStore, + createFakeMemoryProvider, } from "@corbits/knowledge-engine"; -const knowledge = createKnowledgePlane(loadKnowledgeConfig()); -await knowledge.capture({ tenantId, principalId, title, text }); -await knowledge.close(); -``` - -`capture()` and `search()` do **not** check the capability grant. They apply -per-document visibility and block lists, which is not the same question. So if -the caller is acting on behalf of a user rather than as an operator, check it -yourself: - -```ts -import { authorize } from "@intx/authz"; +const knowledge = createKnowledgePlane(undefined, grants, { + documentStore: createFakeDocumentStore(), + memory: createFakeMemoryProvider(), + generate: async (messages) => "…", // wire your inference layer +}); -const decision = await authorize( - grantStore, - principalId, +await knowledge.add({ tenantId, - "knowledge", - "search", - conditionRegistry, -); -if (decision.effect !== "allow") throw new Error("not permitted"); -``` - -Apply the knowledge/vector schema once (idempotent): - -```ts -import { runKnowledgeMigrations } from "@corbits/knowledge-engine/migrations"; -// Reads FTS_LANGUAGE from the environment (default "english") and fails -// loudly if the database was migrated under a different language. -await runKnowledgeMigrations(process.env.KNOWLEDGE_DATABASE_URL); -``` - -## ask() - -`knowledge.ask()` answers a question from retrieved context, in-process — no -HTTP hop, no separate host-side answer synthesis. It is grant-checked -internally, so it is safe to call from anywhere a host has resolved a -principal, even code paths that never go through the mounted HTTP routes: - -```ts -const { text, citations, evidence } = await knowledge.ask({ + principalId, + content: { title: "Note", text: "…" }, +}); +const hits = await knowledge.find({ tenantId, principalId, query: "…" }); +const answer = await knowledge.ask({ tenantId, principalId, - query, - k: 6, // optional, defaults to hybridSearch's default + query: "…", + includeMemory: false, // default }); ``` -1. Checks the capability grant (`authorize(grantStore, principalId, tenantId, -"knowledge", "search", conditionRegistry)`) and throws - `KnowledgeNotPermittedError` unless the effect is an explicit `"allow"` — no - matching grant (`effect: null`) denies too. This runs whether or not the - call came through the HTTP route guard, so it can never be forgotten. -2. Searches as that principal (`hybridSearch`), so per-document visibility and - block lists apply exactly as they do for `search()`. -3. Assembles a grounded context block from the hits' `snippet` text (search - truncates each snippet to ≤240 chars today — that is the MVP grounding - limit), then truncates the assembled block to a character budget. Citation - numbers are sequential among entries actually included in the prompt. -4. Calls the **host-supplied** `generate` function with a system prompt that - instructs answering only from context and refusing explicitly when the - context doesn't contain the answer. -5. Returns the answer text, the citations included in the prompt (matched to - the `[N]` markers the model was asked to use), and the search's evidence - level. - -### The engine owns no generation client - -`ask()` takes generation as an injected function, not as config: +## Ports -```ts -type Generate = (messages: readonly ChatMessage[]) => Promise; - -mountKnowledgeEngine(app, { - config, - grants, - generate: async (messages) => runInferenceSomehow(messages), -}); -``` +| Port | Default | Override | +| --- | --- | --- | +| `DocumentStore` | Engine pgvector | `options.documentStore` / `createFakeDocumentStore()` | +| `SourceProvider[]` | none | `options.sources` — live merge is fail-soft | +| `MemoryProvider` | none | `options.memory` / `createFakeMemoryProvider()` | -This is deliberate. Interchange already has an inference layer -(`@intx/inference`) with provider adapters, tenant-scoped credentials, a retry -policy, audit collection and authz gates. A `fetch` client here would bypass all -of it and take an API key from a raw env var — so hosts wire `generate` to that -layer instead, and credentials stay in the credential store where they belong. +**Live merge (MergeLocalLiveV1):** per-provider timeout/error → `live_timeout` / +`live_error` degrade; dedupe `adapter:externalRef`; optional `sources` filter. -It also keeps the engine transport-free, which is the posture it already takes -on embedding: never in-process, always an endpoint the owner plugs in. +**Memory:** `ask({ includeMemory: true })` recalls when a provider is mounted; +failure → `memory_unavailable`, docs-only. `plane.remember` / `plane.recall` for +host-owned writes (ask never auto-remembers). -Omit `generate` if the host only captures and searches; `ask()` then fails with -a 501 naming what is missing rather than at some later point. +### Adapter packages -Two things that belong in the host's `generate`, learned the hard way: +```ts +// packages/knowledge-adapter-mem0 — DocumentStore (not MemoryProvider) +import { createMem0DocumentStore } from "@corbits/knowledge-adapter-mem0"; -- **Timeouts must be generous for local models.** A cold 10GB model can take - over a minute to page into memory before emitting a token. -- **Use a non-reasoning model.** A reasoning model that exhausts its budget - returns chain-of-thought with empty content, and the host will see an empty - answer. Detect it there and say so. +// packages/knowledge-adapter-supermemory — DocumentStore +import { createSupermemoryDocumentStore } from "@corbits/knowledge-adapter-supermemory"; -## Local development +// Linear SourceProvider lives in sibling repo @corbits/linear +// (https://github.com/corbitsdev/corbits-linear), not this monorepo. +import { + createLinearSourceProvider, + mapLinearWebhook, +} from "@corbits/linear"; +// host owns OAuth + webhook verify; private issues never map to tenant visibility +``` -The SDK is not a server — it mounts onto your app. This repo ships a -`compose.yml` for the backing services so you can develop against it: +## Migrations ```bash -docker compose up -d # pgvector + Ollama + reranker -docker compose exec ollama ollama pull nomic-embed-text -cp .env.example .env -bun install -bun run db:setup # apply the knowledge schema +# KNOWLEDGE_DATABASE_URL required — no DATABASE_URL fallback +bun run db:setup +# or: runKnowledgeMigrations(process.env.KNOWLEDGE_DATABASE_URL) ``` -## Config - -`loadKnowledgeConfig()` reads the environment (see `.env.example`) and returns a -`KnowledgeConfig` — just the vector DB and model endpoints. Hosts that don't -want env-driven config can build the object directly. See `PRODUCT.md` for the -shape and the identity/ACL model, and `IMPLEMENTATION.md` for env vars and -service internals. - -### Reranking - -Reranking is optional (`RERANK_BASE_URL` etc.). Supported API styles are **TEI, -Cohere v2, and Voyage**; retrieval degrades to fusion-only when no base URL is -set. +All tables live under Postgres schema **`knowledge`** +(`knowledge.document`, `knowledge.version`, `knowledge.chunk`, …). Hard cutover +pre-1.0: re-run migrations on a fresh knowledge DB. -Document character budgets (`RERANK_MAX_DOC_CHARS`) and startup validation -against known model token limits apply to the **TEI path only** — TEI rejects -the whole batch if any single document exceeds the cross-encoder's limit, and -the engine's ~700-token chunks routinely exceed `bge-reranker-base`'s 512. -Left unset, the budget is derived from the resolved model's advertised token -limit rather than a single constant (the engine default, `bge-reranker-v2-m3`, -has an 8,192-token limit — over 16× `bge-reranker-base` — so a one-size budget -would either 413 the smaller model or over-truncate the larger one). +## ACL ladder (honest) -The TEI budget also reserves space for the query (the limit is on the -query+document pair). If the query alone leaves less than a useful minimum for -the document, reranking is skipped for that request (logged, reported as -`"rerank_query_too_long"`) rather than truncating the query. +1. Host grants (`knowledge:add` / `knowledge:find`) +2. Per-document visibility (`private` | `principals` | `tenant` + optional blocks) +3. Share sugars on `add` map onto (2) — no second ACL system -`mountKnowledgeEngine` validates the TEI budget against known models at startup -and throws `RerankConfigError` on a mismatch. A replay's `transform_config` can -supply its own rerank endpoint/model; that path is validated the same way at -request time and degrades to fused ranking on a mismatch instead of throwing. +## Docs -Truncation is a real tradeoff: the reranker scores only the head of a chunk -while callers still cite the whole thing, and the char budget is an estimate -(~3 chars/token for prose — denser content like CJK, minified code, or base64 -can still overflow). Lower `RERANK_MAX_DOC_CHARS` for those corpora. +- `PRODUCT.md` — product shape and out-of-scope +- `ARCHITECTURE.md` — design decisions +- `IMPLEMENTATION.md` — env vars, data model, services +- `MIGRATION.md` — hard cutover from capture/search/timeline -### Degrade metrics - -When retrieval degrades (missing embed model, rerank failure, query-too-long, -and similar), the engine records counters you can poll — there is no metrics -port in this package. Export `getDegradeMetricsSnapshot` / -`getAllDegradeMetricsSnapshots` (and optional `configureDegradeMetrics`) and -forward them from the host's own metrics backend. - -## Testing +## Develop ```bash -bun run test # unit suite: bun test ./src -bun run test:coverage +bun install bun run typecheck +bun run test +# adapter packages (DocumentStore backends): +bun test packages/knowledge-adapter-mem0 +bun test packages/knowledge-adapter-supermemory +# Linear tools: sibling repo @corbits/linear ``` -Unit tests are colocated under `src/` and run entirely against mocked -boundaries — no external services required. - -## License - -LGPL-2.1 — see [`LICENSE`](LICENSE). +License: LGPL-2.1 (`LICENSE`). Contributions: `CLA.md`. diff --git a/package.json b/package.json index 444089b..241e641 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@corbits/knowledge-engine", "version": "0.1.2", - "description": "Mountable knowledge capture + search SDK for Interchange hubs", + "description": "Mountable knowledge add/find/ask/recent SDK for Interchange hubs", "exports": { ".": "./src/index.ts", "./migrations": "./src/migrations.ts", diff --git a/packages/knowledge-adapter-mem0/README.md b/packages/knowledge-adapter-mem0/README.md index d843e52..62609b3 100644 --- a/packages/knowledge-adapter-mem0/README.md +++ b/packages/knowledge-adapter-mem0/README.md @@ -1,72 +1,89 @@ -# `@corbits/knowledge-adapter-mem0` +# @corbits/knowledge-adapter-mem0 -Mem0-backed [`MemoryProvider`](https://github.com/corbitsdev/corbits-knowledge-engine) for `@corbits/knowledge-engine`. +**Replaceable DocumentStore** for [@corbits/knowledge-engine](https://github.com/corbitsdev/corbits-knowledge-engine) +backed by the [Mem0 Platform](https://docs.mem0.ai/) HTTP API. -Pure HTTP (`fetch`) against the Mem0 Platform API — **no** `mem0ai` SDK dependency. Vendor code stays out of the knowledge-engine core. +Pure `fetch` — **no** `mem0ai` / vendor SDK. Tenancy is enforced with a +length-prefixed `user_id` (`mapUser`) so free-form ids cannot collide. -## Identity mapping +## Product path: DocumentStore -Mem0 scopes memories by `user_id`. This adapter never sends a bare principal: +Mount Mem0 as `documentStore` so the plane routes `add` / `find` / `recent` +(and `ask` via find) through this store — no Postgres / embed endpoints +required. This is the product integration path (not `MemoryProvider` / +`includeMemory`). ```ts -mapUser(tenantId, principalId) // → `${tenantId}::${principalId}` -``` - -Empty/missing `tenantId` or `principalId` throws. Same principal under two tenants gets two distinct Mem0 users. - -## Usage - -```ts -import { createMem0MemoryProvider } from "@corbits/knowledge-adapter-mem0"; import { createKnowledgePlane } from "@corbits/knowledge-engine"; - -const memory = createMem0MemoryProvider({ - apiKey: process.env.MEM0_API_KEY!, - // baseUrl: "https://api.mem0.ai", // optional - // fetch: customFetch, // optional (tests / proxies) +import { createMem0DocumentStore } from "@corbits/knowledge-adapter-mem0"; + +const knowledge = createKnowledgePlane(undefined, grants, { + documentStore: createMem0DocumentStore({ + apiKey: process.env.MEM0_API_KEY!, + // baseUrl: "https://api.mem0.ai", // optional + }), + generate: myGenerate, // required only for ask() }); -const plane = createKnowledgePlane(db, authz, { - documentStore, - memory, +await knowledge.add({ + tenantId, + principalId, + content: { title: "Prefs", text: "Prefers dark mode" }, }); -await plane.remember({ - tenantId: "acme", - principalId: "user-42", - text: "Prefers TypeScript strict mode", +const { items } = await knowledge.find({ + tenantId, + principalId, + query: "preferences", }); +``` + +Or via mount: -// ask() recalls only when includeMemory: true -const answer = await plane.ask({ - tenantId: "acme", - principalId: "user-42", - query: "What language preferences do I have?", - includeMemory: true, +```ts +mountKnowledgeEngine(app, { + documentStore: createMem0DocumentStore({ apiKey }), + grants, + generate, }); ``` -## Options +## Limitations (honest) -| Option | Required | Description | -| --------- | -------- | ------------------------------------------------ | -| `apiKey` | yes | Mem0 API key (`Authorization: Token …`) | -| `baseUrl` | no | API origin (default `https://api.mem0.ai`) | -| `fetch` | no | Injectable `fetch` for tests / custom transports | +| Area | Behavior | +| --- | --- | +| Isolation | **Principal-bucket only** via `mapUser(tenantId, principalId)`. Each principal has a private Mem0 `user_id`; docs are not shared across principals. | +| Visibility ladder | `visibility` / `share` / `blockPrincipalIds` are **not** enforced by this adapter (metadata at best). For multi-principal or tenant-wide ACL, use the default pgvector store or a store that implements the ladder. | +| `recent` | Always `[]` — Mem0 has no recent-feed API here. | +| `options.memory` | **Never** mount this package as `options.memory`. That port is an ask side-channel; Mem0 as product backend is `documentStore` only. | -## HTTP surface +## What is out of scope -| Op | Method | Path | -| -------- | ------ | ---------------------- | -| remember | POST | `/v3/memories/add/` | -| recall | POST | `/v3/memories/search/` | +- **Not** a tools-shaped source (Linear-style live connectors stay separate). +- **Not** the product path for `MemoryProvider` / `includeMemory`. +- `createMem0MemoryProvider` remains exported for back-compat only; prefer + `createMem0DocumentStore`. -Search filters always include `user_id: mapUser(tenantId, principalId)`. +## Tenant mapping -## Tests +```ts +import { mapUser } from "@corbits/knowledge-adapter-mem0"; -```bash -bun test +mapUser("acme", "alice"); // "4:acme:5:alice" ``` -All tests use a mocked `fetch` — no live network. +Never pass bare `principalId` as Mem0 `user_id`. + +## HTTP surface (thin) + +| Verb | Method | Path | +| ------ | ------ | ------------------------ | +| add | POST | `/v3/memories/add/` | +| find | POST | `/v3/memories/search/` | +| recent | — | empty (API has no feed) | + +Auth: `Authorization: Token `. + +## License + +LGPL-2.1-only (same as the knowledge engine). diff --git a/packages/knowledge-adapter-mem0/package.json b/packages/knowledge-adapter-mem0/package.json index 643940e..11dacec 100644 --- a/packages/knowledge-adapter-mem0/package.json +++ b/packages/knowledge-adapter-mem0/package.json @@ -1,7 +1,8 @@ { "name": "@corbits/knowledge-adapter-mem0", "version": "0.1.0", - "description": "Mem0 MemoryProvider adapter for @corbits/knowledge-engine (pure fetch, no SDK)", + "description": "Mem0 DocumentStore adapter for @corbits/knowledge-engine (pure fetch, no SDK)", + "type": "module", "module": "src/index.ts", "exports": { diff --git a/packages/knowledge-adapter-mem0/src/create-mem0-document-store.test.ts b/packages/knowledge-adapter-mem0/src/create-mem0-document-store.test.ts new file mode 100644 index 0000000..2f49024 --- /dev/null +++ b/packages/knowledge-adapter-mem0/src/create-mem0-document-store.test.ts @@ -0,0 +1,230 @@ +import { describe, expect, it } from "bun:test"; + +import { + createMem0DocumentStore, + parseFindResults, +} from "./create-mem0-document-store.ts"; + +type Captured = { + url: string; + method: string; + headers: Record; + body: unknown; +}; + +function mockFetch( + handler: (req: Captured) => { status?: number; json?: unknown }, +): { fetch: typeof fetch; calls: Captured[] } { + const calls: Captured[] = []; + const fetchImpl = (async ( + input: string | URL | Request, + init?: RequestInit, + ): Promise => { + const url = + typeof input === "string" + ? input + : input instanceof URL + ? input.href + : input.url; + const headers: Record = {}; + if (init?.headers) { + const h = new Headers(init.headers); + h.forEach((v, k) => { + headers[k] = v; + }); + } + let body: unknown; + if (typeof init?.body === "string") { + body = JSON.parse(init.body); + } + const cap: Captured = { + url, + method: init?.method ?? "GET", + headers, + body, + }; + calls.push(cap); + const result = handler(cap); + const status = result.status ?? 200; + const payload = + result.json === undefined ? "" : JSON.stringify(result.json); + return new Response(payload, { + status, + headers: { "Content-Type": "application/json" }, + }); + }) as typeof fetch; + return { fetch: fetchImpl, calls }; +} + +describe("createMem0DocumentStore", () => { + it("rejects missing apiKey", () => { + expect(() => createMem0DocumentStore({ apiKey: "" })).toThrow(/apiKey/); + }); + + it("add posts mapped user_id and returns documentId", async () => { + const { fetch, calls } = mockFetch(() => ({ + status: 200, + json: { event_id: "e1", status: "PENDING" }, + })); + const store = createMem0DocumentStore({ apiKey: "test-key", fetch }); + + const { documentId } = await store.add({ + tenantId: "t1", + principalId: "p1", + title: "Prefs", + text: "Prefers dark mode", + visibility: { mode: "private", principalIds: ["p1"] }, + }); + + expect(documentId).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i, + ); + expect(calls).toHaveLength(1); + const call = calls[0]!; + expect(call.method).toBe("POST"); + expect(call.url).toBe("https://api.mem0.ai/v3/memories/add/"); + const body = call.body as Record; + expect(body.user_id).toBe("2:t1:2:p1"); + expect(body.user_id).not.toBe("p1"); + expect(body.infer).toBe(false); + const messages = body.messages as Array<{ content: string }>; + expect(messages[0]!.content).toContain("# Prefs"); + expect(messages[0]!.content).toContain("Prefers dark mode"); + expect(messages[0]!.content).toContain(documentId); + const meta = body.metadata as Record; + expect(meta.documentId).toBe(documentId); + expect(meta.title).toBe("Prefs"); + }); + + it("find scopes search by mapped user_id and maps hits", async () => { + const { fetch, calls } = mockFetch(() => ({ + status: 200, + json: { + results: [ + { + id: "m1", + memory: "# Home\n\nLives in SF", + score: 0.91, + metadata: { documentId: "doc-1", externalRef: "ref-1" }, + }, + ], + }, + })); + const store = createMem0DocumentStore({ + apiKey: "k", + baseUrl: "https://mem0.example.com/", + fetch, + }); + + const result = await store.find({ + tenantId: "acme", + principalId: "bob", + query: "where do I live?", + limit: 3, + includeEvidence: true, + }); + + expect(calls).toHaveLength(1); + const call = calls[0]!; + expect(call.url).toBe("https://mem0.example.com/v3/memories/search/"); + const body = call.body as Record; + expect(body.filters).toEqual({ user_id: "4:acme:3:bob" }); + expect(body.top_k).toBe(3); + + expect(result.evidence).toBe("weak"); + expect(result.items).toHaveLength(1); + expect(result.items[0]!.documentId).toBe("doc-1"); + expect(result.items[0]!.title).toBe("Home"); + expect(result.items[0]!.snippet).toContain("Lives in SF"); + expect(result.items[0]!.citation.adapter).toBe("mem0"); + expect(result.items[0]!.citation.external_ref).toBe("ref-1"); + }); + + it("add/find reject empty identity", async () => { + const { fetch, calls } = mockFetch(() => ({ status: 200, json: {} })); + const store = createMem0DocumentStore({ apiKey: "k", fetch }); + + await expect( + store.add({ + tenantId: "", + principalId: "p", + title: "t", + text: "x", + visibility: { mode: "tenant" }, + }), + ).rejects.toThrow(/tenantId/); + + await expect( + store.find({ + tenantId: "t", + principalId: "", + query: "q", + }), + ).rejects.toThrow(/principalId/); + + expect(calls).toHaveLength(0); + }); + + it("tenant isolation: same principal different tenants → distinct user_id", async () => { + const { fetch, calls } = mockFetch(() => ({ + status: 200, + json: { results: [] }, + })); + const store = createMem0DocumentStore({ apiKey: "k", fetch }); + + await store.find({ + tenantId: "tenant-a", + principalId: "alice", + query: "prefs", + }); + await store.find({ + tenantId: "tenant-b", + principalId: "alice", + query: "prefs", + }); + + const userIds = calls.map( + (c) => (c.body as { filters: { user_id: string } }).filters.user_id, + ); + expect(userIds).toEqual(["8:tenant-a:5:alice", "8:tenant-b:5:alice"]); + expect(userIds[0]).not.toBe(userIds[1]); + }); + + it("recent returns empty list; close is a no-op", async () => { + const store = createMem0DocumentStore({ + apiKey: "k", + fetch: (async () => new Response("{}")) as unknown as typeof fetch, + }); + + expect( + await store.recent({ + tenantId: "t", + principalId: "p", + }), + ).toEqual([]); + await store.close(); + }); +}); + +describe("parseFindResults", () => { + it("reads results[].memory with metadata documentId", () => { + const items = parseFindResults({ + results: [ + { + memory: "# Note\n\nbody", + score: 0.5, + metadata: { documentId: "d1" }, + }, + ], + }); + expect(items).toHaveLength(1); + expect(items[0]!.documentId).toBe("d1"); + expect(items[0]!.title).toBe("Note"); + }); + + it("handles empty / null", () => { + expect(parseFindResults(null)).toEqual([]); + expect(parseFindResults(undefined)).toEqual([]); + expect(parseFindResults({})).toEqual([]); + }); +}); diff --git a/packages/knowledge-adapter-mem0/src/create-mem0-document-store.ts b/packages/knowledge-adapter-mem0/src/create-mem0-document-store.ts new file mode 100644 index 0000000..a6c1da3 --- /dev/null +++ b/packages/knowledge-adapter-mem0/src/create-mem0-document-store.ts @@ -0,0 +1,264 @@ +import { mapUser } from "./map-user.ts"; +import type { + DocumentStore, + DocumentStoreFindItem, + Mem0ClientOptions, +} from "./types.ts"; + +const DEFAULT_BASE_URL = "https://api.mem0.ai"; +const ADAPTER = "mem0"; + +function normalizeBaseUrl(baseUrl: string): string { + return baseUrl.replace(/\/+$/, ""); +} + +async function readErrorBody(res: Response): Promise { + try { + const text = await res.text(); + return text.length > 500 ? `${text.slice(0, 500)}…` : text; + } catch { + return ""; + } +} + +function requireIdentity(tenantId: string, principalId: string): void { + if (typeof tenantId !== "string" || tenantId.trim() === "") { + throw new Error("Mem0 DocumentStore requires non-empty tenantId"); + } + if (typeof principalId !== "string" || principalId.trim() === "") { + throw new Error("Mem0 DocumentStore requires non-empty principalId"); + } +} + +function encodeDocumentBody(params: { + title: string; + text: string; + documentId: string; + externalRef?: string; + visibilityMode: string; +}): string { + const header = `# ${params.title}`; + const meta = [ + `documentId: ${params.documentId}`, + params.externalRef ? `externalRef: ${params.externalRef}` : null, + `visibility: ${params.visibilityMode}`, + ] + .filter(Boolean) + .join("\n"); + return `${header}\n\n${params.text}\n\n---\n${meta}`; +} + +function parseTitleAndSnippet(text: string): { title: string; snippet: string } { + const lines = text.split("\n"); + if (lines[0]?.startsWith("# ")) { + const title = lines[0].slice(2).trim() || "untitled"; + const rest = lines + .slice(1) + .join("\n") + .replace(/\n---\n[\s\S]*$/, "") + .trim(); + return { + title, + snippet: rest.slice(0, 240) || title, + }; + } + return { + title: text.slice(0, 80) || "untitled", + snippet: text.slice(0, 240), + }; +} + +/** Normalize Mem0 search JSON into DocumentStore find items. */ +export function parseFindResults(raw: unknown): DocumentStoreFindItem[] { + if (raw == null) return []; + + let items: unknown[] = []; + if (Array.isArray(raw)) { + items = raw; + } else if (typeof raw === "object" && raw !== null) { + const obj = raw as Record; + if (Array.isArray(obj.results)) { + items = obj.results; + } else if (Array.isArray(obj.memories)) { + items = obj.memories; + } + } + + const out: DocumentStoreFindItem[] = []; + for (const item of items) { + if (item == null || typeof item !== "object") continue; + const row = item as Record; + const text = + typeof row.memory === "string" + ? row.memory + : typeof row.text === "string" + ? row.text + : null; + if (text == null) continue; + + const idFromRow = + typeof row.id === "string" + ? row.id + : typeof row.memory_id === "string" + ? row.memory_id + : undefined; + const meta = + row.metadata && typeof row.metadata === "object" + ? (row.metadata as Record) + : {}; + const documentId = + (typeof meta.documentId === "string" && meta.documentId) || + idFromRow || + crypto.randomUUID(); + const externalRef = + (typeof meta.externalRef === "string" && meta.externalRef) || documentId; + const { title, snippet } = parseTitleAndSnippet(text); + const score = + typeof row.score === "number" && Number.isFinite(row.score) + ? row.score + : 0.5; + + out.push({ + documentId, + title, + snippet, + score, + kind: "note", + adapter: ADAPTER, + externalRef, + citation: { + adapter: ADAPTER, + external_ref: externalRef, + open: { + type: "document", + id: documentId, + url: `mem0://${documentId}`, + }, + }, + }); + } + return out; +} + +/** + * Create a DocumentStore backed by the Mem0 Platform HTTP API (v3). + * + * Pure fetch — no mem0 SDK. Mount as the plane's durable backend: + * + * ```ts + * createKnowledgePlane(undefined, grants, { + * documentStore: createMem0DocumentStore({ apiKey }), + * }) + * ``` + * + * No local Postgres required. Tenancy is enforced via length-prefixed `user_id` + * (`mapUser`); never pass bare principalId. + */ +export function createMem0DocumentStore( + opts: Mem0ClientOptions, +): DocumentStore { + if (typeof opts.apiKey !== "string" || opts.apiKey.trim() === "") { + throw new Error( + "createMem0DocumentStore: apiKey is required and must be a non-empty string", + ); + } + + const baseUrl = normalizeBaseUrl(opts.baseUrl ?? DEFAULT_BASE_URL); + const doFetch = opts.fetch ?? globalThis.fetch.bind(globalThis); + const authHeader = `Token ${opts.apiKey}`; + + async function mem0Post( + path: string, + body: Record, + ): Promise { + const url = `${baseUrl}${path}`; + const res = await doFetch(url, { + method: "POST", + headers: { + Authorization: authHeader, + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(body), + }); + + if (!res.ok) { + const detail = await readErrorBody(res); + throw new Error( + `Mem0 API ${path} failed: HTTP ${res.status}${detail ? ` — ${detail}` : ""}`, + ); + } + + if (res.status === 204) return undefined; + const text = await res.text(); + if (!text) return undefined; + try { + return JSON.parse(text) as unknown; + } catch { + return undefined; + } + } + + return { + async add(params) { + requireIdentity(params.tenantId, params.principalId); + const userId = mapUser(params.tenantId, params.principalId); + const documentId = crypto.randomUUID(); + const content = encodeDocumentBody({ + title: params.title, + text: params.text, + documentId, + ...(params.externalRef !== undefined + ? { externalRef: params.externalRef } + : {}), + visibilityMode: params.visibility.mode, + }); + const metadata: Record = { + documentId, + title: params.title, + visibility: params.visibility.mode, + }; + if (params.externalRef !== undefined) { + metadata.externalRef = params.externalRef; + } + + // Platform v3 add path (same as legacy MemoryProvider). + await mem0Post("/v3/memories/add/", { + messages: [{ role: "user", content }], + user_id: userId, + infer: false, + metadata, + }); + + return { documentId }; + }, + + async find(params) { + requireIdentity(params.tenantId, params.principalId); + const userId = mapUser(params.tenantId, params.principalId); + const topK = params.limit ?? 8; + const raw = await mem0Post("/v3/memories/search/", { + query: params.query, + filters: { user_id: userId }, + top_k: topK, + }); + const items = parseFindResults(raw); + if (params.includeEvidence) { + return { + items, + evidence: items.length === 0 ? "none" : "weak", + }; + } + return { items }; + }, + + async recent() { + // Query-oriented API; no stable recent timeline in this thin adapter. + return []; + }, + + async close() { + // Stateless HTTP client. + }, + }; +} diff --git a/packages/knowledge-adapter-mem0/src/create-mem0-memory-provider.test.ts b/packages/knowledge-adapter-mem0/src/create-mem0-memory-provider.test.ts index 1e4abf5..b01aca7 100644 --- a/packages/knowledge-adapter-mem0/src/create-mem0-memory-provider.test.ts +++ b/packages/knowledge-adapter-mem0/src/create-mem0-memory-provider.test.ts @@ -17,7 +17,7 @@ function mockFetch( ): { fetch: typeof fetch; calls: Captured[] } { const calls: Captured[] = []; const fetchImpl = (async ( - input: RequestInfo | URL, + input: string | URL | Request, init?: RequestInit, ): Promise => { const url = @@ -86,7 +86,8 @@ describe("createMem0MemoryProvider", () => { "Token test-key", ); const body = call.body as Record; - expect(body.user_id).toBe("t1::p1"); + expect(body.user_id).toBe("2:t1:2:p1"); + expect(body.user_id).not.toBe("p1"); expect(body.messages).toEqual([ { role: "user", content: "Prefers dark mode" }, @@ -124,7 +125,8 @@ describe("createMem0MemoryProvider", () => { const body = call.body as Record; expect(body.query).toBe("where do I live?"); expect(body.top_k).toBe(3); - expect(body.filters).toEqual({ user_id: "acme::bob" }); + expect(body.filters).toEqual({ user_id: "4:acme:3:bob" }); + expect(hits).toEqual([ { text: "Lives in SF", score: 0.91 }, { text: "Works remote", score: 0.7 }, @@ -190,7 +192,8 @@ describe("createMem0MemoryProvider", () => { const userIds = calls.map( (c) => (c.body as { filters: { user_id: string } }).filters.user_id, ); - expect(userIds).toEqual(["tenant-a::alice", "tenant-b::alice"]); + expect(userIds).toEqual(["8:tenant-a:5:alice", "8:tenant-b:5:alice"]); + expect(userIds[0]).not.toBe(userIds[1]); }); }); diff --git a/packages/knowledge-adapter-mem0/src/create-mem0-memory-provider.ts b/packages/knowledge-adapter-mem0/src/create-mem0-memory-provider.ts index 3c95270..739295c 100644 --- a/packages/knowledge-adapter-mem0/src/create-mem0-memory-provider.ts +++ b/packages/knowledge-adapter-mem0/src/create-mem0-memory-provider.ts @@ -1,5 +1,9 @@ +/** + * @deprecated Prefer createMem0DocumentStore and mount as documentStore. + * Thin remember/recall wrapper kept for back-compat only — not the product path. + */ import { mapUser } from "./map-user.ts"; -import type { Mem0MemoryProviderOptions, MemoryProvider } from "./types.ts"; +import type { Mem0ClientOptions, MemoryProvider } from "./types.ts"; const DEFAULT_BASE_URL = "https://api.mem0.ai"; @@ -17,12 +21,10 @@ async function readErrorBody(res: Response): Promise { } /** - * Create a MemoryProvider backed by the Mem0 Platform HTTP API (v3). - * - * Uses pure fetch — no mem0 SDK. Inject `fetch` in tests. + * @deprecated Use createMem0DocumentStore({ apiKey }) as options.documentStore. */ export function createMem0MemoryProvider( - opts: Mem0MemoryProviderOptions, + opts: Mem0ClientOptions, ): MemoryProvider { if (typeof opts.apiKey !== "string" || opts.apiKey.trim() === "") { throw new Error( @@ -56,7 +58,6 @@ export function createMem0MemoryProvider( ); } - // 204 / empty body if (res.status === 204) return undefined; const text = await res.text(); if (!text) return undefined; @@ -73,7 +74,6 @@ export function createMem0MemoryProvider( const body: Record = { messages: [{ role: "user", content: params.text }], user_id: userId, - // Host already decided the fact; store verbatim. infer: false, }; if (params.metadata !== undefined) { @@ -90,7 +90,6 @@ export function createMem0MemoryProvider( filters: { user_id: userId }, top_k: topK, }); - return parseSearchResults(raw); }, }; diff --git a/packages/knowledge-adapter-mem0/src/index.ts b/packages/knowledge-adapter-mem0/src/index.ts index 22a8217..625f9dd 100644 --- a/packages/knowledge-adapter-mem0/src/index.ts +++ b/packages/knowledge-adapter-mem0/src/index.ts @@ -1,6 +1,17 @@ -export type { MemoryProvider, Mem0MemoryProviderOptions } from "./types.ts"; +export type { + DocumentStore, + DocumentStoreAddParams, + DocumentStoreFindItem, + DocumentStoreFindParams, + DocumentStoreFindResult, + DocumentStoreRecentEvent, + DocumentStoreRecentParams, + Mem0ClientOptions, + Mem0MemoryProviderOptions, + MemoryProvider, + VisibilitySpec, +} from "./types.ts"; export { mapUser } from "./map-user.ts"; -export { - createMem0MemoryProvider, - parseSearchResults, -} from "./create-mem0-memory-provider.ts"; +export { createMem0DocumentStore } from "./create-mem0-document-store.ts"; +/** @deprecated Prefer createMem0DocumentStore as options.documentStore. */ +export { createMem0MemoryProvider } from "./create-mem0-memory-provider.ts"; diff --git a/packages/knowledge-adapter-mem0/src/map-user.test.ts b/packages/knowledge-adapter-mem0/src/map-user.test.ts index 5afde6e..6deff8b 100644 --- a/packages/knowledge-adapter-mem0/src/map-user.test.ts +++ b/packages/knowledge-adapter-mem0/src/map-user.test.ts @@ -3,25 +3,35 @@ import { describe, expect, it } from "bun:test"; import { mapUser } from "./map-user.ts"; describe("mapUser", () => { - it("joins tenantId::principalId", () => { - expect(mapUser("tenant-a", "user-1")).toBe("tenant-a::user-1"); + it("length-prefixes tenant and principal", () => { + expect(mapUser("tenant-a", "user-1")).toBe("8:tenant-a:6:user-1"); }); - it("isolates same principal across tenants", () => { + it("isolates the same principal across tenants", () => { const a = mapUser("tenant-a", "alice"); const b = mapUser("tenant-b", "alice"); - expect(a).toBe("tenant-a::alice"); - expect(b).toBe("tenant-b::alice"); expect(a).not.toBe(b); }); it("rejects empty tenantId", () => { expect(() => mapUser("", "alice")).toThrow(/tenantId/); - expect(() => mapUser(" ", "alice")).toThrow(/tenantId/); }); it("rejects empty principalId", () => { - expect(() => mapUser("tenant-a", "")).toThrow(/principalId/); - expect(() => mapUser("tenant-a", " ")).toThrow(/principalId/); + expect(() => mapUser("t", "")).toThrow(/principalId/); + }); + + it("rejects whitespace-only ids", () => { + expect(() => mapUser(" ", "alice")).toThrow(/tenantId/); + expect(() => mapUser("t", " ")).toThrow(/principalId/); + }); + + it("is injective when ids contain delimiter sequences", () => { + // Old `tenant::principal` encoding collided on these pairs. + const a = mapUser("a::b", "c"); + const b = mapUser("a", "b::c"); + expect(a).not.toBe(b); + expect(a).toBe("4:a::b:1:c"); + expect(b).toBe("1:a:4:b::c"); }); }); diff --git a/packages/knowledge-adapter-mem0/src/map-user.ts b/packages/knowledge-adapter-mem0/src/map-user.ts index 6c53e43..683644a 100644 --- a/packages/knowledge-adapter-mem0/src/map-user.ts +++ b/packages/knowledge-adapter-mem0/src/map-user.ts @@ -1,8 +1,9 @@ /** * Map Corbits (tenantId, principalId) → Mem0 user_id. * - * Always `tenantId::principalId` so the same principal in different tenants - * never shares a Mem0 user. Bare principal is forbidden. + * Length-prefixed encoding is injective for any free-form ids that do not + * contain only digits-before-colon collisions: distinct pairs never share a + * user_id even when ids contain `::` or `_`. */ export function mapUser(tenantId: string, principalId: string): string { if (typeof tenantId !== "string" || tenantId.trim() === "") { @@ -15,5 +16,6 @@ export function mapUser(tenantId: string, principalId: string): string { "mapUser: principalId is required and must be a non-empty string", ); } - return `${tenantId}::${principalId}`; + // `${len}:${id}` twice — cannot collide across delimiter injection. + return `${tenantId.length}:${tenantId}:${principalId.length}:${principalId}`; } diff --git a/packages/knowledge-adapter-mem0/src/types.ts b/packages/knowledge-adapter-mem0/src/types.ts index c9a739e..a187ce2 100644 --- a/packages/knowledge-adapter-mem0/src/types.ts +++ b/packages/knowledge-adapter-mem0/src/types.ts @@ -1,7 +1,90 @@ /** - * MemoryProvider port — defined locally so this adapter never imports - * runtime from @corbits/knowledge-engine. Shape matches core ports/types. + * Port shapes — defined locally so this adapter never imports runtime from + * @corbits/knowledge-engine. DocumentStore is the product plug (replaceable + * durable backend). MemoryProvider is a thin legacy shape kept for back-compat. */ + +/** Minimal citation open shape (matches knowledge-engine SearchHitCitation). */ +export type DocumentStoreCitation = { + adapter: string; + external_ref: string; + open: { + type: string; + id: string; + url?: string; + }; +}; + +export type VisibilitySpec = + | { mode: "tenant" } + | { mode: "private"; principalIds: string[] } + | { mode: "principals"; principalIds: string[] }; + +export type DocumentStoreAddParams = { + tenantId: string; + principalId: string; + title: string; + text: string; + visibility: VisibilitySpec; + blockPrincipalIds?: string[]; + attributes?: Record; + externalRef?: string; +}; + +export type DocumentStoreFindParams = { + tenantId: string; + principalId: string; + query: string; + limit?: number; + includeEvidence?: boolean; +}; + +export type DocumentStoreFindItem = { + documentId: string; + title: string; + snippet: string; + score: number; + kind: string; + citation: DocumentStoreCitation; + adapter?: string; + externalRef?: string; + updatedAt?: string; +}; + +export type DocumentStoreFindResult = { + items: DocumentStoreFindItem[]; + evidence?: "strong" | "weak" | "none"; + degraded?: string[]; +}; + +export type DocumentStoreRecentParams = { + tenantId: string; + principalId: string; + limit?: number; +}; + +export type DocumentStoreRecentEvent = { + at: string; + title: string; + source: string; + tenantId: string; + principalId: string; +}; + +/** + * Replaceable durable backend for the knowledge plane (add / find / recent). + * Mount as `options.documentStore` — no local Postgres required. + */ +export type DocumentStore = { + add(params: DocumentStoreAddParams): Promise<{ documentId: string }>; + find(params: DocumentStoreFindParams): Promise; + recent( + params: DocumentStoreRecentParams, + ): Promise; + close(): Promise; +}; + +/** @deprecated Prefer DocumentStore. Thin remember/recall only. */ export type MemoryProvider = { remember(params: { tenantId: string; @@ -17,7 +100,7 @@ export type MemoryProvider = { }): Promise>; }; -export type Mem0MemoryProviderOptions = { +export type Mem0ClientOptions = { /** Mem0 platform API key (sent as `Authorization: Token …`). */ apiKey: string; /** API origin; default `https://api.mem0.ai`. */ @@ -25,3 +108,6 @@ export type Mem0MemoryProviderOptions = { /** Injectable fetch for tests; defaults to global fetch. */ fetch?: typeof fetch; }; + +/** @deprecated Use Mem0ClientOptions */ +export type Mem0MemoryProviderOptions = Mem0ClientOptions; diff --git a/packages/knowledge-adapter-supermemory/README.md b/packages/knowledge-adapter-supermemory/README.md index 03afc7c..9c4a19b 100644 --- a/packages/knowledge-adapter-supermemory/README.md +++ b/packages/knowledge-adapter-supermemory/README.md @@ -1,59 +1,79 @@ -# `@corbits/knowledge-adapter-supermemory` +# @corbits/knowledge-adapter-supermemory -Supermemory adapter for the Corbits Knowledge Engine `MemoryProvider` port. +**Replaceable DocumentStore** for [@corbits/knowledge-engine](https://github.com/corbitsdev/corbits-knowledge-engine) +backed by the [Supermemory](https://supermemory.ai/) HTTP API. -Pure `fetch` HTTP — **no** `supermemory` npm SDK. +Pure `fetch` — **no** vendor SDK. Tenancy is enforced with a length-prefixed +`containerTag` so free-form ids cannot collide. -## Install +## Product path: DocumentStore -```bash -bun add @corbits/knowledge-adapter-supermemory +Mount Supermemory as `documentStore` so the plane routes `add` / `find` / +`recent` (and `ask` via find) through this store — no Postgres / embed +endpoints required. This is the product integration path (not +`MemoryProvider` / `includeMemory`). + +```ts +import { createKnowledgePlane } from "@corbits/knowledge-engine"; +import { createSupermemoryDocumentStore } from "@corbits/knowledge-adapter-supermemory"; + +const knowledge = createKnowledgePlane(undefined, grants, { + documentStore: createSupermemoryDocumentStore({ + apiKey: process.env.SUPERMEMORY_API_KEY!, + }), + generate: myGenerate, +}); ``` -## Usage +Or via mount: ```ts -import { - createSupermemoryMemoryProvider, - containerTag, -} from "@corbits/knowledge-adapter-supermemory"; - -const memory = createSupermemoryMemoryProvider({ - apiKey: process.env.SUPERMEMORY_API_KEY!, - // baseUrl?: "https://api.supermemory.ai" // or self-hosted - // fetch?: myFetch // injectable for tests +mountKnowledgeEngine(app, { + documentStore: createSupermemoryDocumentStore({ apiKey }), + grants, + generate, }); - -// Mount on the knowledge plane -// createKnowledgePlane({ …, memory }) ``` -### Container tags +Find uses `searchMode: "hybrid"` so document retrieval works for the green +plane (add/find/ask), not memories-only personal facts. -Tenant isolation maps to Supermemory `containerTag`: +## Limitations (honest) -``` -t_{tenantId}_u_{principalId} -``` +| Area | Behavior | +| --- | --- | +| Isolation | **Principal-bucket only** via `containerTag(tenantId, principalId)`. Each principal has a private container; docs are not shared across principals. | +| Visibility ladder | `visibility` / `share` / `blockPrincipalIds` are **not** enforced by this adapter. For multi-principal or tenant-wide ACL, use the default pgvector store or a store that implements the ladder. | +| `recent` | Always `[]` — no recent-feed API in this adapter. | +| `options.memory` | **Never** mount this package as `options.memory`. Product backend is `documentStore` only. | -Example: `containerTag("acme", "alice")` → `t_acme_u_alice`. +## What is out of scope -Empty `tenantId` / `principalId` are rejected. +- **Not** a tools-shaped source (Linear-style live connectors stay separate). +- **Not** the product path for `MemoryProvider` / `includeMemory`. +- `createSupermemoryMemoryProvider` remains exported for back-compat only + (memories-only recall); prefer `createSupermemoryDocumentStore`. -### Recall +## Tenant mapping -`recall` always sends `searchMode: "memories"` (extracted facts only). It never -relies on the API default. +```ts +import { containerTag } from "@corbits/knowledge-adapter-supermemory"; -| Method | HTTP | -| -------- | ---------------------------- | -| remember | `POST /v3/documents` | -| recall | `POST /v4/search` | +containerTag("acme", "alice"); // "t4_acme_u5_alice" +``` -## Tests +Never pass bare `principalId` as `containerTag`. -```bash -bun test -``` +## HTTP surface (thin) + +| Verb | Method | Path | Notes | +| ------ | ------ | ----------------- | ------------------------------ | +| add | POST | `/v3/documents` | content + containerTag | +| find | POST | `/v4/search` | `searchMode: "hybrid"` | +| recent | — | empty | API has no recent feed here | + +Auth: `Authorization: Bearer `. + +## License -All network is mocked; no live Supermemory calls. +LGPL-2.1-only (same as the knowledge engine). diff --git a/packages/knowledge-adapter-supermemory/package.json b/packages/knowledge-adapter-supermemory/package.json index d1f68b4..4fc7275 100644 --- a/packages/knowledge-adapter-supermemory/package.json +++ b/packages/knowledge-adapter-supermemory/package.json @@ -1,7 +1,8 @@ { "name": "@corbits/knowledge-adapter-supermemory", "version": "0.1.0", - "description": "Supermemory MemoryProvider adapter for Corbits Knowledge Engine (pure fetch, no vendor SDK)", + "description": "Supermemory DocumentStore adapter for Corbits Knowledge Engine (pure fetch, no vendor SDK)", + "exports": { ".": "./src/index.ts" }, diff --git a/packages/knowledge-adapter-supermemory/src/index.test.ts b/packages/knowledge-adapter-supermemory/src/index.test.ts index 54b949c..d736792 100644 --- a/packages/knowledge-adapter-supermemory/src/index.test.ts +++ b/packages/knowledge-adapter-supermemory/src/index.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, mock } from "bun:test"; import { containerTag, + createSupermemoryDocumentStore, createSupermemoryMemoryProvider, } from "./index.ts"; @@ -13,80 +14,57 @@ function jsonResponse(body: unknown, status = 200): Response { } describe("containerTag", () => { - it("maps tenant + principal to t_{tenant}_u_{principal}", () => { - expect(containerTag("acme", "alice")).toBe("t_acme_u_alice"); - expect(containerTag("org-1", "user-42")).toBe("t_org-1_u_user-42"); + it("maps tenant + principal with length prefixes", () => { + expect(containerTag("acme", "alice")).toBe("t4_acme_u5_alice"); + expect(containerTag("org-1", "user-42")).toBe("t5_org-1_u7_user-42"); }); it("produces distinct tags per tenant for the same principal", () => { const a = containerTag("tenant-a", "user-1"); const b = containerTag("tenant-b", "user-1"); - expect(a).toBe("t_tenant-a_u_user-1"); - expect(b).toBe("t_tenant-b_u_user-1"); + expect(a).toBe("t8_tenant-a_u6_user-1"); + expect(b).toBe("t8_tenant-b_u6_user-1"); expect(a).not.toBe(b); }); + it("is injective when ids contain delimiter sequences", () => { + const a = containerTag("x_u_y", "z"); + const b = containerTag("x", "y_u_z"); + expect(a).not.toBe(b); + expect(a).toBe("t5_x_u_y_u1_z"); + expect(b).toBe("t1_x_u5_y_u_z"); + }); + it("rejects empty tenantId or principalId", () => { expect(() => containerTag("", "alice")).toThrow(/non-empty/); expect(() => containerTag("acme", "")).toThrow(/non-empty/); expect(() => containerTag("", "")).toThrow(/non-empty/); + expect(() => containerTag(" ", "alice")).toThrow(/non-empty/); }); }); -describe("createSupermemoryMemoryProvider", () => { - it("rejects empty identity on remember", async () => { +describe("createSupermemoryDocumentStore", () => { + it("rejects empty identity on add", async () => { const fetchImpl = mock(() => Promise.resolve(jsonResponse({ id: "x" }))); - const provider = createSupermemoryMemoryProvider({ + const store = createSupermemoryDocumentStore({ apiKey: "test-key", fetch: fetchImpl as unknown as typeof fetch, }); await expect( - provider.remember({ + store.add({ tenantId: "", principalId: "alice", + title: "t", text: "hello", - }), - ).rejects.toThrow(/non-empty/); - - await expect( - provider.remember({ - tenantId: "acme", - principalId: "", - text: "hello", + visibility: { mode: "tenant" }, }), ).rejects.toThrow(/non-empty/); expect(fetchImpl).not.toHaveBeenCalled(); }); - it("rejects empty identity on recall", async () => { - const fetchImpl = mock(() => Promise.resolve(jsonResponse({ results: [] }))); - const provider = createSupermemoryMemoryProvider({ - apiKey: "test-key", - fetch: fetchImpl as unknown as typeof fetch, - }); - - await expect( - provider.recall({ - tenantId: "", - principalId: "alice", - query: "prefs", - }), - ).rejects.toThrow(/non-empty/); - - await expect( - provider.recall({ - tenantId: "acme", - principalId: "", - query: "prefs", - }), - ).rejects.toThrow(/non-empty/); - - expect(fetchImpl).not.toHaveBeenCalled(); - }); - - it("remember posts to /v3/documents with containerTag", async () => { + it("add posts to /v3/documents with containerTag and returns documentId", async () => { const fetchImpl = mock((url: string, init?: RequestInit) => { expect(url).toBe("https://api.supermemory.ai/v3/documents"); expect(init?.method).toBe("POST"); @@ -97,31 +75,35 @@ describe("createSupermemoryMemoryProvider", () => { containerTag: string; metadata?: Record; }; - expect(body.content).toBe("prefers dark mode"); - expect(body.containerTag).toBe("t_acme_u_alice"); - expect(body.metadata).toEqual({ source: "chat" }); + expect(body.containerTag).toBe("t4_acme_u5_alice"); + expect(body.content).toContain("# Prefs"); + expect(body.content).toContain("prefers dark mode"); + expect(body.metadata?.title).toBe("Prefs"); return Promise.resolve(jsonResponse({ id: "doc_1", status: "queued" })); }); - const provider = createSupermemoryMemoryProvider({ + const store = createSupermemoryDocumentStore({ apiKey: "test-key", fetch: fetchImpl as unknown as typeof fetch, }); - await provider.remember({ + const { documentId } = await store.add({ tenantId: "acme", principalId: "alice", + title: "Prefs", text: "prefers dark mode", - metadata: { source: "chat" }, + visibility: { mode: "private", principalIds: ["alice"] }, }); + expect(documentId).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i, + ); expect(fetchImpl).toHaveBeenCalledTimes(1); }); - it("recall always sends searchMode: memories (never default)", async () => { + it("find uses hybrid searchMode and maps hits", async () => { const fetchImpl = mock((url: string, init?: RequestInit) => { expect(url).toBe("https://api.supermemory.ai/v4/search"); - expect(init?.method).toBe("POST"); const body = JSON.parse(init?.body as string) as { q: string; containerTag: string; @@ -129,43 +111,45 @@ describe("createSupermemoryMemoryProvider", () => { limit?: number; }; expect(body.q).toBe("preferences"); - expect(body.containerTag).toBe("t_acme_u_alice"); - expect(body.searchMode).toBe("memories"); + expect(body.containerTag).toBe("t4_acme_u5_alice"); + expect(body.searchMode).toBe("hybrid"); expect(body.limit).toBe(3); - // Must not omit searchMode (would fall through to API default). - expect("searchMode" in body).toBe(true); return Promise.resolve( jsonResponse({ results: [ { id: "mem_1", - memory: "User prefers dark mode", + chunk: "# Theme\n\nUser prefers dark mode", similarity: 0.92, + metadata: { documentId: "d1" }, }, ], }), ); }); - const provider = createSupermemoryMemoryProvider({ + const store = createSupermemoryDocumentStore({ apiKey: "test-key", fetch: fetchImpl as unknown as typeof fetch, }); - const hits = await provider.recall({ + const result = await store.find({ tenantId: "acme", principalId: "alice", query: "preferences", limit: 3, + includeEvidence: true, }); - expect(hits).toEqual([ - { text: "User prefers dark mode", score: 0.92 }, - ]); + expect(result.evidence).toBe("weak"); + expect(result.items).toHaveLength(1); + expect(result.items[0]!.documentId).toBe("d1"); + expect(result.items[0]!.title).toBe("Theme"); + expect(result.items[0]!.citation.adapter).toBe("supermemory"); expect(fetchImpl).toHaveBeenCalledTimes(1); }); - it("scopes container tags distinctly per tenant on remember", async () => { + it("scopes container tags distinctly per tenant on add", async () => { const tags: string[] = []; const fetchImpl = mock((_url: string, init?: RequestInit) => { const body = JSON.parse(init?.body as string) as { containerTag: string }; @@ -173,47 +157,80 @@ describe("createSupermemoryMemoryProvider", () => { return Promise.resolve(jsonResponse({ id: "x" })); }); - const provider = createSupermemoryMemoryProvider({ + const store = createSupermemoryDocumentStore({ apiKey: "test-key", fetch: fetchImpl as unknown as typeof fetch, }); - await provider.remember({ + await store.add({ tenantId: "tenant-a", principalId: "user-1", + title: "a", text: "fact a", + visibility: { mode: "tenant" }, }); - await provider.remember({ + await store.add({ tenantId: "tenant-b", principalId: "user-1", + title: "b", text: "fact b", + visibility: { mode: "tenant" }, }); - expect(tags).toEqual(["t_tenant-a_u_user-1", "t_tenant-b_u_user-1"]); + expect(tags).toEqual(["t8_tenant-a_u6_user-1", "t8_tenant-b_u6_user-1"]); }); - it("uses custom baseUrl when provided", async () => { - const fetchImpl = mock((url: string) => { - expect(url).toBe("http://localhost:6767/v4/search"); - return Promise.resolve(jsonResponse({ results: [] })); + it("recent returns empty; close is a no-op", async () => { + const store = createSupermemoryDocumentStore({ + apiKey: "test-key", + fetch: mock(() => Promise.resolve(jsonResponse({}))) as unknown as typeof fetch, + }); + expect( + await store.recent({ tenantId: "t", principalId: "u" }), + ).toEqual([]); + await store.close(); + }); + + it("throws when apiKey is empty", () => { + expect(() => createSupermemoryDocumentStore({ apiKey: "" })).toThrow( + /apiKey/, + ); + }); +}); + +describe("createSupermemoryMemoryProvider (legacy)", () => { + it("recall always sends searchMode: memories", async () => { + const fetchImpl = mock((url: string, init?: RequestInit) => { + expect(url).toBe("https://api.supermemory.ai/v4/search"); + const body = JSON.parse(init?.body as string) as { + searchMode: string; + }; + expect(body.searchMode).toBe("memories"); + return Promise.resolve( + jsonResponse({ + results: [ + { + id: "mem_1", + memory: "User prefers dark mode", + similarity: 0.92, + }, + ], + }), + ); }); const provider = createSupermemoryMemoryProvider({ apiKey: "test-key", - baseUrl: "http://localhost:6767/", fetch: fetchImpl as unknown as typeof fetch, }); - await provider.recall({ - tenantId: "t", - principalId: "u", - query: "q", + const hits = await provider.recall({ + tenantId: "acme", + principalId: "alice", + query: "preferences", + limit: 3, }); - }); - it("throws when apiKey is empty", () => { - expect(() => - createSupermemoryMemoryProvider({ apiKey: "" }), - ).toThrow(/apiKey/); + expect(hits).toEqual([{ text: "User prefers dark mode", score: 0.92 }]); }); }); diff --git a/packages/knowledge-adapter-supermemory/src/index.ts b/packages/knowledge-adapter-supermemory/src/index.ts index 21e9707..5c67079 100644 --- a/packages/knowledge-adapter-supermemory/src/index.ts +++ b/packages/knowledge-adapter-supermemory/src/index.ts @@ -1,12 +1,94 @@ /** - * Supermemory MemoryProvider adapter. + * Supermemory DocumentStore adapter (replaceable durable backend). * - * Pure fetch HTTP against the Supermemory REST API — no vendor SDK. - * MemoryProvider is defined locally so this package never imports the - * knowledge-engine runtime. + * Pure fetch HTTP — no vendor SDK. Port shapes are defined locally so this + * package never imports the knowledge-engine runtime. + * + * Product path: createSupermemoryDocumentStore → mount as options.documentStore. + * MemoryProvider factory is back-compat only. + */ + +/** Minimal citation open shape (matches knowledge-engine SearchHitCitation). */ +export type DocumentStoreCitation = { + adapter: string; + external_ref: string; + open: { + type: string; + id: string; + url?: string; + }; +}; + +export type VisibilitySpec = + | { mode: "tenant" } + | { mode: "private"; principalIds: string[] } + | { mode: "principals"; principalIds: string[] }; + +export type DocumentStoreAddParams = { + tenantId: string; + principalId: string; + title: string; + text: string; + visibility: VisibilitySpec; + blockPrincipalIds?: string[]; + attributes?: Record; + externalRef?: string; +}; + +export type DocumentStoreFindParams = { + tenantId: string; + principalId: string; + query: string; + limit?: number; + includeEvidence?: boolean; +}; + +export type DocumentStoreFindItem = { + documentId: string; + title: string; + snippet: string; + score: number; + kind: string; + citation: DocumentStoreCitation; + adapter?: string; + externalRef?: string; + updatedAt?: string; +}; + +export type DocumentStoreFindResult = { + items: DocumentStoreFindItem[]; + evidence?: "strong" | "weak" | "none"; + degraded?: string[]; +}; + +export type DocumentStoreRecentParams = { + tenantId: string; + principalId: string; + limit?: number; +}; + +export type DocumentStoreRecentEvent = { + at: string; + title: string; + source: string; + tenantId: string; + principalId: string; +}; + +/** + * Replaceable durable backend for the knowledge plane (add / find / recent). + * Mount as `options.documentStore` — no local Postgres required. */ +export type DocumentStore = { + add(params: DocumentStoreAddParams): Promise<{ documentId: string }>; + find(params: DocumentStoreFindParams): Promise; + recent( + params: DocumentStoreRecentParams, + ): Promise; + close(): Promise; +}; -/** Local port contract (mirrors knowledge-engine MemoryProvider). */ +/** @deprecated Prefer DocumentStore. Thin remember/recall only. */ export type MemoryProvider = { remember(params: { tenantId: string; @@ -23,21 +105,28 @@ export type MemoryProvider = { }; const DEFAULT_BASE_URL = "https://api.supermemory.ai"; +const ADAPTER = "supermemory"; /** * Map tenant + principal to a Supermemory containerTag. - * Format: `t_{tenantId}_u_{principalId}` + * Length-prefixed so free-form ids cannot collide across delimiter injection: + * `t{len}_{tenant}_u{len}_{principal}` */ export function containerTag(tenantId: string, principalId: string): string { - if (tenantId === "" || principalId === "") { + if (typeof tenantId !== "string" || tenantId.trim() === "") { throw new Error( "containerTag requires non-empty tenantId and principalId", ); } - return `t_${tenantId}_u_${principalId}`; + if (typeof principalId !== "string" || principalId.trim() === "") { + throw new Error( + "containerTag requires non-empty tenantId and principalId", + ); + } + return `t${tenantId.length}_${tenantId}_u${principalId.length}_${principalId}`; } -export type SupermemoryMemoryProviderOpts = { +export type SupermemoryClientOpts = { apiKey: string; /** API root (no trailing slash). Default: https://api.supermemory.ai */ baseUrl?: string; @@ -45,10 +134,18 @@ export type SupermemoryMemoryProviderOpts = { fetch?: typeof fetch; }; +/** @deprecated Use SupermemoryClientOpts */ +export type SupermemoryMemoryProviderOpts = SupermemoryClientOpts; + function requireIdentity(tenantId: string, principalId: string): void { - if (tenantId === "" || principalId === "") { + if (typeof tenantId !== "string" || tenantId.trim() === "") { throw new Error( - "tenantId and principalId must be non-empty strings", + "Supermemory DocumentStore requires non-empty tenantId and principalId", + ); + } + if (typeof principalId !== "string" || principalId.trim() === "") { + throw new Error( + "Supermemory DocumentStore requires non-empty tenantId and principalId", ); } } @@ -69,21 +166,220 @@ async function readErrorBody(res: Response): Promise { } } +function encodeContent(params: { + title: string; + text: string; + documentId: string; + externalRef?: string; + visibilityMode: string; +}): string { + const header = `# ${params.title}`; + const meta = [ + `documentId: ${params.documentId}`, + params.externalRef ? `externalRef: ${params.externalRef}` : null, + `visibility: ${params.visibilityMode}`, + ] + .filter(Boolean) + .join("\n"); + return `${header}\n\n${params.text}\n\n---\n${meta}`; +} + +function parseTitleAndSnippet(text: string): { title: string; snippet: string } { + const lines = text.split("\n"); + if (lines[0]?.startsWith("# ")) { + const title = lines[0].slice(2).trim() || "untitled"; + const rest = lines + .slice(1) + .join("\n") + .replace(/\n---\n[\s\S]*$/, "") + .trim(); + return { + title, + snippet: rest.slice(0, 240) || title, + }; + } + return { + title: text.slice(0, 80) || "untitled", + snippet: text.slice(0, 240), + }; +} + /** - * Create a MemoryProvider backed by Supermemory (v3 documents + v4 search). + * Create a DocumentStore backed by Supermemory (v3 documents + v4 search). + * + * Pure fetch — no vendor SDK. Mount as the plane's durable backend: * - * recall always sends `searchMode: "memories"` so only extracted facts are - * returned — never hybrid/documents defaults. + * ```ts + * createKnowledgePlane(undefined, grants, { + * documentStore: createSupermemoryDocumentStore({ apiKey }), + * }) + * ``` + * + * Find uses hybrid search (documents + chunks) so the store can answer + * retrieval for add/find/ask — not memories-only personal facts. + */ +export function createSupermemoryDocumentStore( + opts: SupermemoryClientOpts, +): DocumentStore { + const baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\/$/, ""); + const fetchImpl = opts.fetch ?? globalThis.fetch; + const { apiKey } = opts; + + if (typeof apiKey !== "string" || apiKey.trim() === "") { + throw new Error( + "createSupermemoryDocumentStore requires a non-empty apiKey", + ); + } + + return { + async add(params) { + requireIdentity(params.tenantId, params.principalId); + const tag = containerTag(params.tenantId, params.principalId); + const documentId = crypto.randomUUID(); + const content = encodeContent({ + title: params.title, + text: params.text, + documentId, + ...(params.externalRef !== undefined + ? { externalRef: params.externalRef } + : {}), + visibilityMode: params.visibility.mode, + }); + const metadata: Record = { + documentId, + title: params.title, + visibility: params.visibility.mode, + }; + if (params.externalRef !== undefined) { + metadata.externalRef = params.externalRef; + } + + const res = await fetchImpl(`${baseUrl}/v3/documents`, { + method: "POST", + headers: jsonHeaders(apiKey), + body: JSON.stringify({ + content, + containerTag: tag, + metadata, + }), + }); + if (!res.ok) { + const snippet = await readErrorBody(res); + throw new Error( + `Supermemory add failed HTTP ${res.status}: ${snippet}`, + ); + } + return { documentId }; + }, + + async find(params) { + requireIdentity(params.tenantId, params.principalId); + const tag = containerTag(params.tenantId, params.principalId); + const body: Record = { + q: params.query, + containerTag: tag, + // Hybrid retrieval for store replacement (not memories-only facts). + searchMode: "hybrid", + }; + if (params.limit !== undefined) { + body.limit = params.limit; + } + + const res = await fetchImpl(`${baseUrl}/v4/search`, { + method: "POST", + headers: jsonHeaders(apiKey), + body: JSON.stringify(body), + }); + if (!res.ok) { + const snippet = await readErrorBody(res); + throw new Error( + `Supermemory find failed HTTP ${res.status}: ${snippet}`, + ); + } + + const data = (await res.json()) as { + results?: Array<{ + id?: string; + memory?: string; + chunk?: string; + content?: string; + similarity?: number; + metadata?: Record; + }>; + }; + + const results = data.results ?? []; + const items: DocumentStoreFindItem[] = []; + for (const r of results) { + const text = r.memory ?? r.chunk ?? r.content ?? ""; + if (text === "") continue; + const meta = r.metadata ?? {}; + const documentId = + (typeof meta.documentId === "string" && meta.documentId) || + (typeof r.id === "string" && r.id) || + crypto.randomUUID(); + const externalRef = + (typeof meta.externalRef === "string" && meta.externalRef) || + documentId; + const { title, snippet } = parseTitleAndSnippet(text); + const score = + typeof r.similarity === "number" && Number.isFinite(r.similarity) + ? r.similarity + : 0.5; + items.push({ + documentId, + title, + snippet, + score, + kind: "note", + adapter: ADAPTER, + externalRef, + citation: { + adapter: ADAPTER, + external_ref: externalRef, + open: { + type: "document", + id: documentId, + url: `supermemory://${documentId}`, + }, + }, + }); + } + + if (params.includeEvidence) { + return { + items, + evidence: items.length === 0 ? "none" : "weak", + }; + } + return { items }; + }, + + async recent() { + return []; + }, + + async close() { + // Stateless HTTP client. + }, + }; +} + +/** + * @deprecated Prefer createSupermemoryDocumentStore as options.documentStore. + * Thin remember/recall kept for back-compat; not the product path. */ export function createSupermemoryMemoryProvider( - opts: SupermemoryMemoryProviderOpts, + opts: SupermemoryClientOpts, ): MemoryProvider { const baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\/$/, ""); const fetchImpl = opts.fetch ?? globalThis.fetch; const { apiKey } = opts; - if (!apiKey) { - throw new Error("createSupermemoryMemoryProvider requires a non-empty apiKey"); + if (typeof apiKey !== "string" || apiKey.trim() === "") { + throw new Error( + "createSupermemoryMemoryProvider requires a non-empty apiKey", + ); } return { @@ -117,7 +413,7 @@ export function createSupermemoryMemoryProvider( const body: Record = { q: params.query, containerTag: tag, - // Always memories — never rely on API default. + // Legacy memories-only path for personal-fact recall. searchMode: "memories", }; if (params.limit !== undefined) { diff --git a/packages/knowledge-source-linear/README.md b/packages/knowledge-source-linear/README.md deleted file mode 100644 index 2821390..0000000 --- a/packages/knowledge-source-linear/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# @corbits/knowledge-source-linear - -Thin `SourceProvider` mapper for Linear. **Host owns OAuth, webhook signature -verification, and cron/reconciliation** — this package authenticates nothing. - -## What this is - -- `createLinearSourceProvider` — optional live search (`searchLive`) against the - Linear GraphQL API using a host-supplied access token. -- Webhook mappers (`mapIssueCreated` / `mapIssueUpdated` / `mapIssueRemoved`, or - `mapLinearWebhook`) — turn Linear issue webhook payloads into `AdaptedDocument` - shapes ready for `knowledge.capture()`. - -## What the host does - -1. **OAuth / tokens** — obtain and refresh Linear access tokens; pass - `accessToken` into the provider factory. -2. **Webhook verify** — validate Linear webhook signatures before calling a - mapper; never trust raw body bytes without verification. -3. **Cron / backfill** — schedule reconciliation pulls if needed; call capture - with mapped documents on a schedule. -4. **Capture** — call `knowledge.capture({ adapter: "linear", document })` (or - the HTTP capture route) with the mapped document. - -## Visibility rules (overshare guard) - -| Linear issue | Mapped visibility | -| --- | --- | -| Private (`private: true` or `team.private: true`) | `private` (single principal) or `principals` (creator + assignee + subscribers only). **Never `tenant`.** | -| Team-visible | `tenant` (company-brain default). Never `source_acl`. | - -Actor kind on all sync writes is always `adapter` — never the webhook installer's -human identity. - -## Usage - -```ts -import { - createLinearSourceProvider, - mapLinearWebhook, -} from "@corbits/knowledge-source-linear"; - -// Live search (host injects token; inject fetch in tests) -const linear = createLinearSourceProvider({ - accessToken: process.env.LINEAR_TOKEN!, - teamId: "optional-team-filter", -}); -// plane options.sources = [linear] - -// Webhook path (host already verified the signature) -const mapped = mapLinearWebhook(payload); -if (mapped) { - await knowledge.capture({ - adapter: "linear", - document: mapped.document, - }); -} -``` - -## Tests - -```bash -bun test -``` - -All network is mocked; fixtures under `fixtures/` drive golden AdaptedDocument -assertions. diff --git a/packages/knowledge-source-linear/fixtures/issue-created.json b/packages/knowledge-source-linear/fixtures/issue-created.json deleted file mode 100644 index 3ce4ef4..0000000 --- a/packages/knowledge-source-linear/fixtures/issue-created.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "action": "create", - "type": "Issue", - "createdAt": "2026-03-01T12:00:00.000Z", - "url": "https://linear.app/acme/issue/CL-100", - "data": { - "id": "issue-uuid-100", - "identifier": "CL-100", - "title": "Wire Linear SourceProvider", - "description": "Thin mapper package; host owns OAuth and webhooks.", - "url": "https://linear.app/acme/issue/CL-100", - "priority": 2, - "teamId": "team-uuid-1", - "creatorId": "user-alice", - "assigneeId": "user-bob", - "subscriberIds": ["user-alice", "user-bob", "user-carol"], - "stateId": "state-todo", - "createdAt": "2026-03-01T12:00:00.000Z", - "updatedAt": "2026-03-01T12:00:00.000Z", - "private": false, - "team": { - "id": "team-uuid-1", - "key": "CL", - "name": "Corbits", - "private": false - }, - "creator": { "id": "user-alice", "name": "Alice" }, - "assignee": { "id": "user-bob", "name": "Bob" }, - "state": { "id": "state-todo", "name": "Todo", "type": "unstarted" } - } -} diff --git a/packages/knowledge-source-linear/fixtures/issue-removed.json b/packages/knowledge-source-linear/fixtures/issue-removed.json deleted file mode 100644 index 7482009..0000000 --- a/packages/knowledge-source-linear/fixtures/issue-removed.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "action": "remove", - "type": "Issue", - "createdAt": "2026-03-03T16:00:00.000Z", - "url": "https://linear.app/acme/issue/CL-100", - "data": { - "id": "issue-uuid-100", - "identifier": "CL-100", - "title": "Wire Linear SourceProvider (done)", - "description": "Thin mapper package; host owns OAuth and webhooks. Shipped.", - "url": "https://linear.app/acme/issue/CL-100", - "priority": 2, - "teamId": "team-uuid-1", - "creatorId": "user-alice", - "assigneeId": "user-bob", - "subscriberIds": ["user-alice", "user-bob"], - "stateId": "state-done", - "createdAt": "2026-03-01T12:00:00.000Z", - "updatedAt": "2026-03-03T16:00:00.000Z", - "private": false, - "team": { - "id": "team-uuid-1", - "key": "CL", - "name": "Corbits", - "private": false - }, - "creator": { "id": "user-alice", "name": "Alice" }, - "assignee": { "id": "user-bob", "name": "Bob" }, - "state": { "id": "state-done", "name": "Done", "type": "completed" } - } -} diff --git a/packages/knowledge-source-linear/fixtures/issue-updated.json b/packages/knowledge-source-linear/fixtures/issue-updated.json deleted file mode 100644 index 1e4ac10..0000000 --- a/packages/knowledge-source-linear/fixtures/issue-updated.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "action": "update", - "type": "Issue", - "createdAt": "2026-03-02T09:30:00.000Z", - "url": "https://linear.app/acme/issue/CL-100", - "updatedFrom": { - "title": "Wire Linear SourceProvider", - "updatedAt": "2026-03-01T12:00:00.000Z" - }, - "data": { - "id": "issue-uuid-100", - "identifier": "CL-100", - "title": "Wire Linear SourceProvider (done)", - "description": "Thin mapper package; host owns OAuth and webhooks. Shipped.", - "url": "https://linear.app/acme/issue/CL-100", - "priority": 2, - "teamId": "team-uuid-1", - "creatorId": "user-alice", - "assigneeId": "user-bob", - "subscriberIds": ["user-alice", "user-bob"], - "stateId": "state-done", - "createdAt": "2026-03-01T12:00:00.000Z", - "updatedAt": "2026-03-02T09:30:00.000Z", - "private": false, - "team": { - "id": "team-uuid-1", - "key": "CL", - "name": "Corbits", - "private": false - }, - "creator": { "id": "user-alice", "name": "Alice" }, - "assignee": { "id": "user-bob", "name": "Bob" }, - "state": { "id": "state-done", "name": "Done", "type": "completed" } - } -} diff --git a/packages/knowledge-source-linear/fixtures/private-issue-created.json b/packages/knowledge-source-linear/fixtures/private-issue-created.json deleted file mode 100644 index 6823f26..0000000 --- a/packages/knowledge-source-linear/fixtures/private-issue-created.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "action": "create", - "type": "Issue", - "createdAt": "2026-03-01T14:00:00.000Z", - "url": "https://linear.app/acme/issue/CL-PRIV-1", - "data": { - "id": "issue-uuid-priv-1", - "identifier": "CL-PRIV-1", - "title": "Confidential hiring plan", - "description": "Private team issue — must not become tenant-visible.", - "url": "https://linear.app/acme/issue/CL-PRIV-1", - "priority": 1, - "teamId": "team-private-hr", - "creatorId": "user-alice", - "assigneeId": "user-dave", - "subscriberIds": ["user-alice", "user-dave"], - "stateId": "state-todo", - "createdAt": "2026-03-01T14:00:00.000Z", - "updatedAt": "2026-03-01T14:00:00.000Z", - "private": true, - "team": { - "id": "team-private-hr", - "key": "HR", - "name": "HR Private", - "private": true - }, - "creator": { "id": "user-alice", "name": "Alice" }, - "assignee": { "id": "user-dave", "name": "Dave" }, - "state": { "id": "state-todo", "name": "Todo", "type": "unstarted" } - } -} diff --git a/packages/knowledge-source-linear/fixtures/private-issue-solo.json b/packages/knowledge-source-linear/fixtures/private-issue-solo.json deleted file mode 100644 index 7d8434b..0000000 --- a/packages/knowledge-source-linear/fixtures/private-issue-solo.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "action": "create", - "type": "Issue", - "createdAt": "2026-03-01T15:00:00.000Z", - "data": { - "id": "issue-uuid-priv-solo", - "identifier": "CL-PRIV-2", - "title": "Solo private note", - "description": "Only the creator is a principal.", - "priority": 4, - "teamId": "team-private-hr", - "creatorId": "user-alice", - "assigneeId": "user-alice", - "subscriberIds": ["user-alice"], - "private": true, - "team": { - "id": "team-private-hr", - "private": true - }, - "creator": { "id": "user-alice", "name": "Alice" }, - "assignee": { "id": "user-alice", "name": "Alice" } - } -} diff --git a/packages/knowledge-source-linear/package.json b/packages/knowledge-source-linear/package.json deleted file mode 100644 index 9950045..0000000 --- a/packages/knowledge-source-linear/package.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "name": "@corbits/knowledge-source-linear", - "version": "0.1.0", - "description": "Thin Linear SourceProvider mapper for @corbits/knowledge-engine. Host owns OAuth, webhooks, and cron.", - "license": "LGPL-2.1-only", - "type": "module", - "module": "src/index.ts", - "exports": { - ".": "./src/index.ts" - }, - "engines": { - "bun": ">=1.2.0" - }, - "scripts": { - "typecheck": "tsc --noEmit", - "test": "bun test ./src ./fixtures" - }, - "devDependencies": { - "@types/bun": "latest", - "typescript": "^5.9.0" - }, - "files": [ - "src", - "!src/**/*.test.ts", - "README.md", - "LICENSE" - ], - "publishConfig": { - "access": "public" - }, - "keywords": [ - "linear", - "knowledge", - "source-provider", - "corbits" - ] -} diff --git a/packages/knowledge-source-linear/src/hash.ts b/packages/knowledge-source-linear/src/hash.ts deleted file mode 100644 index d8f6ee5..0000000 --- a/packages/knowledge-source-linear/src/hash.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { createHash } from "node:crypto"; - -function sortValue(value: unknown): unknown { - if (Array.isArray(value)) { - return value.map(sortValue); - } - if (value !== null && typeof value === "object") { - const sorted: Record = {}; - for (const key of Object.keys(value as Record).sort()) { - sorted[key] = sortValue((value as Record)[key]); - } - return sorted; - } - return value; -} - -export function stableStringify(value: unknown): string { - return JSON.stringify(sortValue(value)); -} - -/** NOOP key for AdaptedDocument — same logical content → same hash. */ -export function contentHash(parts: { - title: string; - kind: string; - externalRef: string; - attributes: Record; - chunkTexts: readonly string[]; -}): string { - const raw = [ - parts.title, - parts.kind, - parts.externalRef, - stableStringify(parts.attributes), - parts.chunkTexts.join(""), - ].join(" "); - return createHash("sha256").update(raw).digest("hex"); -} diff --git a/packages/knowledge-source-linear/src/index.ts b/packages/knowledge-source-linear/src/index.ts deleted file mode 100644 index 606eaba..0000000 --- a/packages/knowledge-source-linear/src/index.ts +++ /dev/null @@ -1,35 +0,0 @@ -/** - * @corbits/knowledge-source-linear - * - * Thin Linear SourceProvider + webhook → AdaptedDocument mappers. - * Host owns OAuth, webhook signature verification, and cron. - */ - -export { createLinearSourceProvider } from "./provider.ts"; -export { - mapIssueCreated, - mapIssueUpdated, - mapIssueRemoved, - mapLinearWebhook, - mapIssueToAdaptedDocument, - ADAPTER, -} from "./map-webhook.ts"; -export { - mapIssueVisibility, - collectPrincipalIds, - isPrivateIssue, -} from "./visibility.ts"; -export { contentHash, stableStringify } from "./hash.ts"; - -export type { - LiveSearchItem, - SourceProvider, - AdaptedDocument, - VisibilitySpec, - LinearIssueData, - LinearWebhookEvent, - LinearWebhookAction, - MappedWebhookResult, - CreateLinearSourceProviderOpts, - FetchLike, -} from "./types.ts"; diff --git a/packages/knowledge-source-linear/src/map-webhook.test.ts b/packages/knowledge-source-linear/src/map-webhook.test.ts deleted file mode 100644 index 66b7e92..0000000 --- a/packages/knowledge-source-linear/src/map-webhook.test.ts +++ /dev/null @@ -1,145 +0,0 @@ -import { describe, expect, it } from "bun:test"; -import { readFileSync } from "node:fs"; -import { join } from "node:path"; -import { - mapIssueCreated, - mapIssueRemoved, - mapIssueUpdated, - mapLinearWebhook, -} from "./map-webhook.ts"; -import type { AdaptedDocument, LinearWebhookEvent } from "./types.ts"; - -const FIXTURES = join(import.meta.dir, "..", "fixtures"); - -function loadFixture(name: string): LinearWebhookEvent { - const raw = readFileSync(join(FIXTURES, name), "utf8"); - return JSON.parse(raw) as LinearWebhookEvent; -} - -function assertAdapterActor(doc: AdaptedDocument) { - expect(doc.actor.kind).toBe("adapter"); - expect(doc.actor.principalId).toBeUndefined(); -} - -function assertNeverSourceAcl(doc: AdaptedDocument) { - expect((doc.visibility as { mode: string }).mode).not.toBe("source_acl"); -} - -describe("fixture → AdaptedDocument goldens", () => { - it("mapIssueCreated: team-visible issue → tenant visibility, adapter actor", () => { - const event = loadFixture("issue-created.json"); - const doc = mapIssueCreated(event); - - expect(doc.kind).toBe("issue"); - expect(doc.title).toBe("Wire Linear SourceProvider"); - expect(doc.externalRef).toBe("CL-100"); - expect(doc.visibility).toEqual({ mode: "tenant" }); - assertNeverSourceAcl(doc); - assertAdapterActor(doc); - expect(doc.chunks).toHaveLength(1); - expect(doc.chunks[0]?.ordinal).toBe(0); - expect(doc.chunks[0]?.text).toContain("Wire Linear SourceProvider"); - expect(doc.chunks[0]?.text).toContain("Thin mapper package"); - expect(doc.attributes?.linear_id).toBe("issue-uuid-100"); - expect(doc.attributes?.identifier).toBe("CL-100"); - expect(doc.attributes?.removed).toBeUndefined(); - expect(doc.contentHash).toMatch(/^[a-f0-9]{64}$/); - expect(doc.entityHints.length).toBeGreaterThanOrEqual(1); - }); - - it("mapIssueUpdated: reflects new title/description and new contentHash", () => { - const created = mapIssueCreated(loadFixture("issue-created.json")); - const updated = mapIssueUpdated(loadFixture("issue-updated.json")); - - expect(updated.title).toBe("Wire Linear SourceProvider (done)"); - expect(updated.externalRef).toBe("CL-100"); - expect(updated.visibility).toEqual({ mode: "tenant" }); - assertAdapterActor(updated); - expect(updated.attributes?.state_name).toBe("Done"); - expect(updated.chunks[0]?.text).toContain("Shipped"); - expect(updated.contentHash).not.toBe(created.contentHash); - }); - - it("mapIssueRemoved: empty chunks, removed attribute, same externalRef", () => { - const doc = mapIssueRemoved(loadFixture("issue-removed.json")); - - expect(doc.externalRef).toBe("CL-100"); - expect(doc.title).toBe("Wire Linear SourceProvider (done)"); - expect(doc.chunks).toEqual([]); - expect(doc.attributes?.removed).toBe(true); - expect(doc.visibility).toEqual({ mode: "tenant" }); - assertAdapterActor(doc); - expect(doc.contentHash).toMatch(/^[a-f0-9]{64}$/); - }); - - it("overshare guard: private multi-principal issue never maps to tenant", () => { - const doc = mapIssueCreated(loadFixture("private-issue-created.json")); - - expect(doc.visibility.mode).not.toBe("tenant"); - expect(doc.visibility.mode).toBe("principals"); - expect(doc.visibility.principalIds).toEqual( - expect.arrayContaining(["user-alice", "user-dave"]), - ); - expect(doc.visibility.principalIds).toHaveLength(2); - assertNeverSourceAcl(doc); - assertAdapterActor(doc); - expect(doc.externalRef).toBe("CL-PRIV-1"); - }); - - it("overshare guard: private solo issue → private mode, never tenant", () => { - const doc = mapIssueCreated(loadFixture("private-issue-solo.json")); - - expect(doc.visibility.mode).toBe("private"); - expect(doc.visibility.mode).not.toBe("tenant"); - expect(doc.visibility.principalIds).toEqual(["user-alice"]); - assertAdapterActor(doc); - }); - - it("private via team.private alone (no issue.private) still not tenant", () => { - const event = loadFixture("private-issue-created.json"); - // Simulate team-private without top-level private flag - const data = { - ...event.data, - private: false, - team: { ...event.data.team, private: true }, - }; - const doc = mapIssueCreated({ ...event, data }); - expect(doc.visibility.mode).not.toBe("tenant"); - expect(["private", "principals"]).toContain(doc.visibility.mode); - }); -}); - -describe("mapLinearWebhook dispatcher", () => { - it("dispatches create/update/remove", () => { - const c = mapLinearWebhook(loadFixture("issue-created.json")); - expect(c?.action).toBe("create"); - expect(c?.document.externalRef).toBe("CL-100"); - - const u = mapLinearWebhook(loadFixture("issue-updated.json")); - expect(u?.action).toBe("update"); - expect(u?.document.title).toContain("done"); - - const r = mapLinearWebhook(loadFixture("issue-removed.json")); - expect(r?.action).toBe("remove"); - expect(r?.document.attributes?.removed).toBe(true); - }); - - it("returns null for non-Issue types", () => { - const event = loadFixture("issue-created.json"); - expect(mapLinearWebhook({ ...event, type: "Comment" })).toBeNull(); - }); - - it("returns null for unknown actions", () => { - const event = loadFixture("issue-created.json"); - expect(mapLinearWebhook({ ...event, action: "restore" })).toBeNull(); - }); -}); - -describe("deterministic goldens (stable contentHash)", () => { - it("same fixture maps to identical contentHash twice", () => { - const a = mapIssueCreated(loadFixture("issue-created.json")); - const b = mapIssueCreated(loadFixture("issue-created.json")); - expect(a.contentHash).toBe(b.contentHash); - expect(a).toEqual(b); - }); -}); diff --git a/packages/knowledge-source-linear/src/map-webhook.ts b/packages/knowledge-source-linear/src/map-webhook.ts deleted file mode 100644 index f83c718..0000000 --- a/packages/knowledge-source-linear/src/map-webhook.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { contentHash } from "./hash.ts"; -import type { - AdaptedDocument, - LinearIssueData, - LinearWebhookAction, - LinearWebhookEvent, - MappedWebhookResult, -} from "./types.ts"; -import { mapIssueVisibility } from "./visibility.ts"; - -const ADAPTER = "linear"; -const KIND_ISSUE = "issue"; - -function externalRef(data: LinearIssueData): string { - return data.identifier?.trim() || data.id; -} - -function issueTitle(data: LinearIssueData): string { - const t = data.title?.trim(); - return t && t.length > 0 ? t : externalRef(data); -} - -function issueUrl(data: LinearIssueData, eventUrl?: string): string | undefined { - return data.url ?? eventUrl ?? undefined; -} - -function buildAttributes( - data: LinearIssueData, - opts: { removed?: boolean } = {}, -): Record { - const attrs: Record = { - linear_id: data.id, - }; - if (data.identifier != null) attrs.identifier = data.identifier; - if (data.priority != null) attrs.priority = data.priority; - if (data.teamId != null) attrs.team_id = data.teamId; - if (data.stateId != null) attrs.state_id = data.stateId; - if (data.state?.name != null) attrs.state_name = data.state.name; - if (data.state?.type != null) attrs.state_type = data.state.type; - if (data.assigneeId != null) attrs.assignee_id = data.assigneeId; - if (data.creatorId != null) attrs.creator_id = data.creatorId; - const url = issueUrl(data); - if (url != null) attrs.url = url; - if (opts.removed) attrs.removed = true; - return attrs; -} - -function buildChunks(data: LinearIssueData, removed: boolean): Array<{ - ordinal: number; - text: string; -}> { - if (removed) return []; - const parts: string[] = []; - const title = issueTitle(data); - parts.push(title); - const desc = data.description?.trim(); - if (desc) parts.push(desc); - const text = parts.join("\n\n"); - if (!text) return []; - return [{ ordinal: 0, text }]; -} - -function entityHints(data: LinearIssueData): unknown[] { - const hints: unknown[] = []; - const assigneeName = data.assignee?.name; - const assigneeId = data.assigneeId ?? data.assignee?.id; - if (assigneeId) { - hints.push({ - kind: "person", - identifier: assigneeId, - ...(assigneeName ? { label: assigneeName } : {}), - }); - } - const creatorName = data.creator?.name; - const creatorId = data.creatorId ?? data.creator?.id; - if (creatorId && creatorId !== assigneeId) { - hints.push({ - kind: "person", - identifier: creatorId, - ...(creatorName ? { label: creatorName } : {}), - }); - } - return hints; -} - -/** - * Core mapper: Linear issue data → AdaptedDocument. - * Actor is always `adapter` for sync writes (never webhook installer identity). - */ -export function mapIssueToAdaptedDocument( - data: LinearIssueData, - opts: { removed?: boolean } = {}, -): AdaptedDocument { - const removed = opts.removed === true; - const title = issueTitle(data); - const ref = externalRef(data); - const attributes = buildAttributes(data, { removed }); - const chunks = buildChunks(data, removed); - const visibility = mapIssueVisibility(data); - - const doc: AdaptedDocument = { - kind: KIND_ISSUE, - title, - externalRef: ref, - visibility, - entityHints: entityHints(data), - chunks, - // Sync writes always attribute to the adapter, never the installer. - actor: { kind: "adapter" }, - contentHash: contentHash({ - title, - kind: KIND_ISSUE, - externalRef: ref, - attributes, - chunkTexts: chunks.map((c) => c.text), - }), - attributes, - }; - return doc; -} - -export function mapIssueCreated(event: LinearWebhookEvent): AdaptedDocument { - return mapIssueToAdaptedDocument(event.data, { removed: false }); -} - -export function mapIssueUpdated(event: LinearWebhookEvent): AdaptedDocument { - return mapIssueToAdaptedDocument(event.data, { removed: false }); -} - -export function mapIssueRemoved(event: LinearWebhookEvent): AdaptedDocument { - return mapIssueToAdaptedDocument(event.data, { removed: true }); -} - -/** - * Dispatch by webhook action. Returns null for non-Issue types or unknown actions. - */ -export function mapLinearWebhook( - event: LinearWebhookEvent, -): MappedWebhookResult | null { - const type = (event.type ?? "Issue").toLowerCase(); - if (type !== "issue") return null; - - const action = event.action as LinearWebhookAction | string; - if (action === "create") { - return { action: "create", document: mapIssueCreated(event) }; - } - if (action === "update") { - return { action: "update", document: mapIssueUpdated(event) }; - } - if (action === "remove") { - return { action: "remove", document: mapIssueRemoved(event) }; - } - return null; -} - -export { ADAPTER }; diff --git a/packages/knowledge-source-linear/src/provider.test.ts b/packages/knowledge-source-linear/src/provider.test.ts deleted file mode 100644 index 532cbc0..0000000 --- a/packages/knowledge-source-linear/src/provider.test.ts +++ /dev/null @@ -1,207 +0,0 @@ -import { describe, expect, it, mock } from "bun:test"; -import { createLinearSourceProvider } from "./provider.ts"; - -type FetchFn = NonNullable< - Parameters[0]["fetch"] ->; - -function jsonResponse(body: unknown, status = 200): Response { - return new Response(JSON.stringify(body), { - status, - headers: { "content-type": "application/json" }, - }); -} - -function asFetch(fn: (...args: Parameters) => Promise): FetchFn { - return fn as FetchFn; -} - -describe("createLinearSourceProvider", () => { - it("id is linear", () => { - const provider = createLinearSourceProvider({ - accessToken: "test-token", - fetch: asFetch(() => Promise.resolve(jsonResponse({ data: {} }))), - }); - expect(provider.id).toBe("linear"); - }); - - it("searchLive maps GraphQL nodes to LiveSearchItem (mocked, no network)", async () => { - const fetchMock = mock( - asFetch((url, init) => { - expect(String(url)).toBe("https://api.linear.app/graphql"); - expect(init?.method).toBe("POST"); - const headers = init?.headers as Record; - expect(headers.authorization).toBe("Bearer test-token"); - const body = JSON.parse(String(init?.body)); - expect(body.variables.term).toBe("ports"); - expect(body.variables.first).toBe(5); - - return Promise.resolve( - jsonResponse({ - data: { - searchIssues: { - nodes: [ - { - id: "uuid-1", - identifier: "CL-1", - title: "ports foundation", - description: "DocumentStore + SourceProvider", - url: "https://linear.app/x/issue/CL-1", - updatedAt: "2026-03-01T00:00:00.000Z", - team: { id: "team-a" }, - }, - { - id: "uuid-2", - identifier: "CL-2", - title: "unrelated", - description: "something else", - url: "https://linear.app/x/issue/CL-2", - team: { id: "team-a" }, - }, - ], - }, - }, - }), - ); - }), - ); - - const provider = createLinearSourceProvider({ - accessToken: "test-token", - fetch: fetchMock as FetchFn, - }); - - const hits = await provider.searchLive!({ - query: "ports", - tenantId: "t1", - principalId: "p1", - limit: 5, - }); - - expect(fetchMock).toHaveBeenCalledTimes(1); - expect(hits).toHaveLength(2); - expect(hits[0]).toMatchObject({ - adapter: "linear", - externalRef: "CL-1", - title: "ports foundation", - kind: "issue", - snippet: "DocumentStore + SourceProvider", - updatedAt: "2026-03-01T00:00:00.000Z", - }); - expect(hits[0]?.citation).toEqual({ - adapter: "linear", - external_ref: "CL-1", - open: { - type: "issue", - id: "CL-1", - url: "https://linear.app/x/issue/CL-1", - }, - }); - expect(hits[0]!.score).toBeGreaterThan(hits[1]!.score); - }); - - it("searchLive filters by teamId when set", async () => { - const fetchMock = mock( - asFetch(() => - Promise.resolve( - jsonResponse({ - data: { - searchIssues: { - nodes: [ - { - id: "a", - identifier: "CL-1", - title: "in team", - team: { id: "team-keep" }, - }, - { - id: "b", - identifier: "CL-2", - title: "other team", - team: { id: "team-drop" }, - }, - ], - }, - }, - }), - ), - ), - ); - - const provider = createLinearSourceProvider({ - accessToken: "tok", - teamId: "team-keep", - fetch: fetchMock as FetchFn, - }); - - const hits = await provider.searchLive!({ - query: "team", - tenantId: "t", - principalId: "p", - }); - expect(hits).toHaveLength(1); - expect(hits[0]?.externalRef).toBe("CL-1"); - }); - - it("searchLive uses custom baseUrl", async () => { - const fetchMock = mock( - asFetch((url) => { - expect(String(url)).toBe("https://example.test/graphql"); - return Promise.resolve( - jsonResponse({ data: { searchIssues: { nodes: [] } } }), - ); - }), - ); - - const provider = createLinearSourceProvider({ - accessToken: "tok", - baseUrl: "https://example.test/graphql", - fetch: fetchMock as FetchFn, - }); - - const hits = await provider.searchLive!({ - query: "x", - tenantId: "t", - principalId: "p", - }); - expect(hits).toEqual([]); - }); - - it("searchLive throws on HTTP error (no silent empty)", async () => { - const provider = createLinearSourceProvider({ - accessToken: "tok", - fetch: asFetch(() => - Promise.resolve(new Response("nope", { status: 401 })), - ), - }); - - await expect( - provider.searchLive!({ - query: "x", - tenantId: "t", - principalId: "p", - }), - ).rejects.toThrow(/401/); - }); - - it("searchLive throws on GraphQL errors", async () => { - const provider = createLinearSourceProvider({ - accessToken: "tok", - fetch: asFetch(() => - Promise.resolve( - jsonResponse({ - errors: [{ message: "rate limited" }], - }), - ), - ), - }); - - await expect( - provider.searchLive!({ - query: "x", - tenantId: "t", - principalId: "p", - }), - ).rejects.toThrow(/rate limited/); - }); -}); diff --git a/packages/knowledge-source-linear/src/provider.ts b/packages/knowledge-source-linear/src/provider.ts deleted file mode 100644 index 95db57e..0000000 --- a/packages/knowledge-source-linear/src/provider.ts +++ /dev/null @@ -1,133 +0,0 @@ -import type { - CreateLinearSourceProviderOpts, - LiveSearchItem, - SourceProvider, -} from "./types.ts"; - -const DEFAULT_BASE_URL = "https://api.linear.app/graphql"; -const ADAPTER_ID = "linear"; - -const SEARCH_ISSUES_QUERY = ` -query SearchIssues($term: String!, $first: Int) { - searchIssues(term: $term, first: $first) { - nodes { - id - identifier - title - description - url - updatedAt - team { - id - } - } - } -} -`; - -type LinearSearchNode = { - id: string; - identifier?: string | null; - title?: string | null; - description?: string | null; - url?: string | null; - updatedAt?: string | null; - team?: { id?: string } | null; -}; - -type LinearSearchResponse = { - data?: { - searchIssues?: { - nodes?: LinearSearchNode[]; - }; - }; - errors?: Array<{ message: string }>; -}; - -function snippetFrom(node: LinearSearchNode): string { - const desc = node.description?.trim(); - if (desc && desc.length > 0) { - return desc.length > 240 ? `${desc.slice(0, 237)}...` : desc; - } - return node.title?.trim() || node.identifier || node.id; -} - -function scoreForRank(index: number, total: number): number { - if (total <= 1) return 1; - return 1 - index / total; -} - -/** - * Thin Linear SourceProvider. Host supplies accessToken; OAuth and token - * refresh live outside this package. - */ -export function createLinearSourceProvider( - opts: CreateLinearSourceProviderOpts, -): SourceProvider { - const fetchFn = opts.fetch ?? globalThis.fetch; - const baseUrl = opts.baseUrl ?? DEFAULT_BASE_URL; - const teamId = opts.teamId; - - return { - id: ADAPTER_ID, - async searchLive(params): Promise { - const limit = params.limit ?? 8; - const res = await fetchFn(baseUrl, { - method: "POST", - headers: { - "content-type": "application/json", - authorization: `Bearer ${opts.accessToken}`, - }, - body: JSON.stringify({ - query: SEARCH_ISSUES_QUERY, - variables: { term: params.query, first: limit }, - }), - }); - - if (!res.ok) { - throw new Error( - `Linear GraphQL HTTP ${res.status}: ${await res.text()}`, - ); - } - - const body = (await res.json()) as LinearSearchResponse; - if (body.errors?.length) { - throw new Error( - `Linear GraphQL errors: ${body.errors.map((e) => e.message).join("; ")}`, - ); - } - - let nodes = body.data?.searchIssues?.nodes ?? []; - if (teamId) { - nodes = nodes.filter((n) => n.team?.id === teamId); - } - nodes = nodes.slice(0, limit); - - return nodes.map((node, i) => { - const externalRef = node.identifier?.trim() || node.id; - const title = node.title?.trim() || externalRef; - const item: LiveSearchItem = { - adapter: ADAPTER_ID, - externalRef, - title, - snippet: snippetFrom(node), - score: scoreForRank(i, nodes.length), - kind: "issue", - citation: { - adapter: ADAPTER_ID, - external_ref: externalRef, - open: { - type: "issue", - id: externalRef, - ...(node.url ? { url: node.url } : {}), - }, - }, - }; - if (node.updatedAt) { - item.updatedAt = node.updatedAt; - } - return item; - }); - }, - }; -} diff --git a/packages/knowledge-source-linear/src/types.ts b/packages/knowledge-source-linear/src/types.ts deleted file mode 100644 index 92abe0a..0000000 --- a/packages/knowledge-source-linear/src/types.ts +++ /dev/null @@ -1,108 +0,0 @@ -/** - * Local port types matching @corbits/knowledge-engine SourceProvider / - * AdaptedDocument contracts. Defined here so this package has no hard - * dependency on the engine (plugin boundary). - */ - -export type LiveSearchItem = { - adapter: string; - externalRef: string; - title: string; - snippet: string; - score: number; - kind: string; - citation: { - adapter: string; - external_ref: string; - open: { type: string; id: string; url?: string }; - }; - updatedAt?: string; -}; - -export type SourceProvider = { - readonly id: string; - searchLive?(params: { - query: string; - tenantId: string; - principalId: string; - limit?: number; - }): Promise; -}; - -export type VisibilitySpec = { - mode: "private" | "tenant" | "principals"; - principalIds?: string[]; -}; - -/** Capture-ready document shape (AdaptedDocument-ish). */ -export type AdaptedDocument = { - kind: string; - title: string; - externalRef: string; - visibility: VisibilitySpec; - entityHints: unknown[]; - chunks: Array<{ ordinal: number; text: string }>; - actor: { kind: "adapter" | "human"; principalId?: string }; - contentHash: string; - attributes?: Record; -}; - -/** Linear issue fields we read from webhooks / GraphQL (subset). */ -export type LinearIssueData = { - id: string; - identifier?: string | null; - title?: string | null; - description?: string | null; - url?: string | null; - priority?: number | null; - teamId?: string | null; - creatorId?: string | null; - assigneeId?: string | null; - subscriberIds?: string[] | null; - stateId?: string | null; - updatedAt?: string | null; - createdAt?: string | null; - /** Explicit private flag when present on the payload. */ - private?: boolean | null; - team?: { - id?: string; - key?: string | null; - name?: string | null; - private?: boolean | null; - } | null; - creator?: { id?: string; name?: string | null } | null; - assignee?: { id?: string; name?: string | null } | null; - state?: { id?: string; name?: string | null; type?: string | null } | null; -}; - -export type LinearWebhookAction = "create" | "update" | "remove"; - -export type LinearWebhookEvent = { - action: LinearWebhookAction | string; - type?: string; - data: LinearIssueData; - url?: string; - createdAt?: string; - updatedFrom?: Record; -}; - -export type MappedWebhookResult = { - action: LinearWebhookAction; - document: AdaptedDocument; -}; - -/** Minimal fetch shape so tests/mocks need not implement full Fetch API. */ -export type FetchLike = ( - input: string | URL | Request, - init?: RequestInit, -) => Promise; - -export type CreateLinearSourceProviderOpts = { - accessToken: string; - /** Optional team filter for live search. */ - teamId?: string; - /** Injectable fetch (tests mock Linear GraphQL). */ - fetch?: FetchLike; - /** Default https://api.linear.app/graphql */ - baseUrl?: string; -}; diff --git a/packages/knowledge-source-linear/src/visibility.test.ts b/packages/knowledge-source-linear/src/visibility.test.ts deleted file mode 100644 index 3b8f935..0000000 --- a/packages/knowledge-source-linear/src/visibility.test.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { describe, expect, it } from "bun:test"; -import { - collectPrincipalIds, - isPrivateIssue, - mapIssueVisibility, -} from "./visibility.ts"; -import type { LinearIssueData } from "./types.ts"; - -const base: LinearIssueData = { - id: "i1", - identifier: "CL-1", - title: "t", - creatorId: "c1", - assigneeId: "a1", - subscriberIds: ["c1", "a1", "s1"], -}; - -describe("mapIssueVisibility", () => { - it("team-visible → tenant, never source_acl", () => { - const v = mapIssueVisibility({ - ...base, - private: false, - team: { private: false }, - }); - expect(v).toEqual({ mode: "tenant" }); - }); - - it("private multi-principal → principals (never tenant)", () => { - const v = mapIssueVisibility({ ...base, private: true }); - expect(v.mode).toBe("principals"); - expect(v.mode).not.toBe("tenant"); - expect(v.principalIds).toEqual(["c1", "a1", "s1"]); - }); - - it("private single principal → private mode", () => { - const v = mapIssueVisibility({ - id: "i2", - creatorId: "solo", - assigneeId: "solo", - subscriberIds: ["solo"], - private: true, - }); - expect(v).toEqual({ mode: "private", principalIds: ["solo"] }); - }); - - it("team.private without issue.private is still private", () => { - expect( - isPrivateIssue({ - id: "i3", - private: false, - team: { private: true }, - }), - ).toBe(true); - const v = mapIssueVisibility({ - id: "i3", - creatorId: "c", - private: false, - team: { private: true }, - }); - expect(v.mode).not.toBe("tenant"); - }); -}); - -describe("collectPrincipalIds", () => { - it("dedupes creator/assignee/subscribers and nested objects", () => { - const ids = collectPrincipalIds({ - id: "x", - creatorId: "a", - assigneeId: "b", - subscriberIds: ["a", "c"], - creator: { id: "a" }, - assignee: { id: "b" }, - }); - expect(ids.sort()).toEqual(["a", "b", "c"]); - }); -}); diff --git a/packages/knowledge-source-linear/src/visibility.ts b/packages/knowledge-source-linear/src/visibility.ts deleted file mode 100644 index b22142f..0000000 --- a/packages/knowledge-source-linear/src/visibility.ts +++ /dev/null @@ -1,54 +0,0 @@ -import type { LinearIssueData, VisibilitySpec } from "./types.ts"; - -/** - * Collect principal ids known from the issue payload. - * Only ids present on the webhook/GraphQL object — never invent team rosters. - */ -export function collectPrincipalIds(data: LinearIssueData): string[] { - const ids = new Set(); - if (data.creatorId) ids.add(data.creatorId); - if (data.assigneeId) ids.add(data.assigneeId); - if (data.creator?.id) ids.add(data.creator.id); - if (data.assignee?.id) ids.add(data.assignee.id); - for (const id of data.subscriberIds ?? []) { - if (id) ids.add(id); - } - return [...ids]; -} - -/** - * Private when the issue or its team is marked private. - * Linear private teams must not expand to tenant-wide visibility. - */ -export function isPrivateIssue(data: LinearIssueData): boolean { - if (data.private === true) return true; - if (data.team?.private === true) return true; - return false; -} - -/** - * Map Linear issue visibility → KE VisibilitySpec. - * - * Rules: - * 1. Private issues → `private` (one principal) or `principals` (creator / - * assignee / subscribers only). NEVER `tenant` (overshare guard). - * 2. Team-visible → `tenant` (company-brain default). Never `source_acl` - * (no aspirational ACL level without a read path). - */ -export function mapIssueVisibility(data: LinearIssueData): VisibilitySpec { - const principalIds = collectPrincipalIds(data); - - if (isPrivateIssue(data)) { - if (principalIds.length <= 1) { - const spec: VisibilitySpec = { mode: "private" }; - if (principalIds.length === 1) { - spec.principalIds = principalIds; - } - return spec; - } - return { mode: "principals", principalIds }; - } - - // Team-visible company knowledge. Prefer tenant; never source_acl. - return { mode: "tenant" }; -} diff --git a/packages/knowledge-source-linear/tsconfig.json b/packages/knowledge-source-linear/tsconfig.json deleted file mode 100644 index 7f98225..0000000 --- a/packages/knowledge-source-linear/tsconfig.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "compilerOptions": { - "lib": ["ESNext"], - "target": "ESNext", - "module": "ESNext", - "moduleResolution": "bundler", - "moduleDetection": "force", - "allowImportingTsExtensions": true, - "verbatimModuleSyntax": true, - "noEmit": true, - "strict": true, - "skipLibCheck": true, - "noUncheckedIndexedAccess": true, - "exactOptionalPropertyTypes": true, - "noFallthroughCasesInSwitch": true, - "forceConsistentCasingInFileNames": true, - "types": ["bun"] - }, - "include": ["src", "fixtures"] -} diff --git a/src/config.ts b/src/config.ts index c4c91ba..8a3f468 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,11 +1,11 @@ /** - * Knowledge / vector plane config — the core capture + search engine. + * Knowledge / vector plane config — the core engine behind add/find/recent. * - * This is the low-level engine config consumed by the DB client and the - * capture/search/transform services. The SDK's mount-level config - * (`KnowledgeConfig`, see mount-config.ts) carries this as its `knowledge` - * sub-object. There is no standalone server here — the SDK mounts onto a host - * Interchange app, so there is no port, service token, or process entrypoint. + * This is the low-level engine config consumed by the DB client and internal + * services. The SDK's mount-level config (`KnowledgeConfig`, see + * mount-config.ts) carries this as its `knowledge` sub-object. There is no + * standalone server here — the SDK mounts onto a host Interchange app, so there + * is no port, service token, or process entrypoint. */ export type EngineConfig = { databaseUrl: string; diff --git a/src/index.ts b/src/index.ts index 777ece4..dcf43ba 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,6 @@ /** - * @corbits/knowledge-engine — a knowledge capture + search engine you mount - * onto an Interchange hub. + * @corbits/knowledge-engine — a knowledge add / find / ask / recent engine you + * mount onto an Interchange hub. * * The host owns auth, tenancy, grants, and the process. This SDK reads the * request principal off the Interchange context and (optionally) taps the @@ -58,6 +58,9 @@ export type { KnowledgePlane, KnowledgePlaneOptions, KnowledgeRecentParams, + KnowledgeRecallItem, + KnowledgeRecallParams, + KnowledgeRememberParams, KnowledgeShare, SearchHit, TextExtractor, @@ -65,7 +68,7 @@ export type { VisibilitySpec, } from "./knowledge.ts"; export { KnowledgeError, KnowledgeNotPermittedError } from "./knowledge.ts"; -// Ports (M2) — pluggable storage + live sources; MemoryProvider type stub for M3 +// Ports — pluggable storage, live sources, and optional memory export type { DocumentStore, DocumentStoreAddParams, @@ -78,6 +81,7 @@ export type { MemoryProvider, SourceProvider, } from "./ports/types.ts"; + export { createFakeDocumentStore, createFakeMemoryProvider, @@ -130,9 +134,12 @@ export type MountKnowledgeEngineOptions = { textExtractor?: TextExtractor; /** Override durable storage (default: engine pgvector store). */ documentStore?: DocumentStore; - /** Live source connectors (merge wired in CL-5227). */ + /** Live source connectors merged into find/ask (fail-soft). */ sources?: SourceProvider[]; - /** Memory port accepted for wiring; product in M3. */ + /** + * Optional ask side-channel only (`includeMemory`). Not a DocumentStore + * replacement — vendor backends mount as `documentStore`. + */ memory?: MemoryProvider; }; diff --git a/src/knowledge.ts b/src/knowledge.ts index 11592d8..8d15b19 100644 --- a/src/knowledge.ts +++ b/src/knowledge.ts @@ -1,8 +1,8 @@ /** - * Knowledge plane backed by the engine's pgvector Postgres. Wraps the capture - * and hybrid-search services directly — no HTTP hop. + * Knowledge plane: green surface add / find / ask / recent. * - * Green surface: add / find / ask / recent. + * One product path — always store-backed. Default store is the engine's + * pgvector DocumentStore; hosts inject Mem0 / Supermemory / fakes the same way. */ import { authorize } from "@intx/authz"; @@ -200,6 +200,8 @@ export type FindItem = { score: number; kind: string; citation: SearchHit["citation"]; + /** ISO timestamp for merge recency when the store provides it. */ + updatedAt?: string; }; export type FindResult = { @@ -373,16 +375,18 @@ export type KnowledgePlaneOptions = { textExtractor?: TextExtractor; /** * Override durable storage. When set, the plane does not open Postgres or - * call embed/rerank endpoints — useful for fakes and alternate backends. + * call embed/rerank endpoints — use for fakes and replaceable backends + * (Mem0, Supermemory). When omitted, the default engine DocumentStore is used. */ documentStore?: DocumentStore; /** - * Live source connectors. Wired into find/ask merge in CL-5227; accepted - * here so mounts can declare them early. + * Live source connectors (tools-shaped). Merged into find/ask via + * MergeLocalLiveV1; not a DocumentStore replacement. */ sources?: SourceProvider[]; /** - * Memory port type accepted for mount wiring; remember/recall product is M3. + * Optional personal-memory side channel for ask(includeMemory). + * Not how you swap durable backends — use documentStore for that. */ memory?: MemoryProvider; }; @@ -499,12 +503,13 @@ function resolveShareAndVisibility(params: KnowledgeAddParams): { /** * Build a knowledge plane. * + * One product path: every plane is store-backed. When `options.documentStore` + * is omitted, the default pgvector engine is wrapped as that store. Hosts + * inject Mem0 / Supermemory / fakes the same way — no second plane implementation. + * * - `grants` is required for `ask()` (in-process capability check). Standalone - * add/find callers may omit it — same as #8's out-of-band plane. - * - Rerank config is validated at construction (same as mount) when using the - * default Postgres-backed store. - * - Pass `options.documentStore` to skip Postgres entirely (fakes / overrides). - * When a store is provided, `config` may be omitted. + * add/find callers may omit it. + * - Rerank config is validated at construction when using the default store. * - Pass `options.sources` for live SourceProviders; find/ask merge via * MergeLocalLiveV1 (fail-soft, 800ms timeout, prefer-local dedupe). */ @@ -513,16 +518,18 @@ export function createKnowledgePlane( grants?: GrantConfig, options: KnowledgePlaneOptions = {}, ): KnowledgePlane { - if (options.documentStore) { - return createPlaneFromStore(options.documentStore, grants, options); - } - if (!config) { - throw new KnowledgeError( - 500, - "KnowledgeConfig is required when documentStore is not provided", - ); - } - return createPlaneFromEngine(config, grants, options); + const store = + options.documentStore ?? + (() => { + if (!config) { + throw new KnowledgeError( + 500, + "KnowledgeConfig is required when documentStore is not provided", + ); + } + return createEngineDocumentStore(config); + })(); + return createPlaneFromStore(store, grants, options); } function wantsLocalChannel(sources: string[] | undefined): boolean { @@ -543,6 +550,7 @@ function findItemsToMergeChannel( score: item.score, kind: item.kind, citation: item.citation, + ...(item.updatedAt !== undefined ? { updatedAt: item.updatedAt } : {}), })); } @@ -613,6 +621,8 @@ async function collectLiveItems(params: { function mergeToFindResult(params: { localItems: FindItem[]; localDegraded?: DegradeFlag[]; + /** Hybrid evidence from the local channel when no live items were active. */ + localEvidence?: HybridSearchResult["evidence"]; liveItems: MergeChannelItem[]; liveDegraded: DegradeFlag[]; limit: number; @@ -641,9 +651,22 @@ function mergeToFindResult(params: { ]; if (params.includeEvidence) { + // Preserve hybrid strong/weak when live did not contribute hits; mixed or + // live-only merges stay conservatively "weak". + const liveContributed = params.liveItems.length > 0; + let evidence: HybridSearchResult["evidence"]; + if (items.length === 0) { + evidence = "none"; + } else if (!liveContributed && params.localEvidence) { + evidence = params.localEvidence; + } else if (!liveContributed) { + evidence = "weak"; + } else { + evidence = "weak"; + } return { items, - evidence: items.length === 0 ? "none" : "weak", + evidence, ...(degraded.length > 0 ? { degraded } : {}), }; } @@ -722,7 +745,7 @@ function makeRememberRecall(options: KnowledgePlaneOptions): { }; } -/** Plane backed by an injected DocumentStore (fake or host override). */ +/** Plane backed by a DocumentStore. Store owns tenancy and document ACL. */ function createPlaneFromStore( store: DocumentStore, grants: GrantConfig | undefined, @@ -735,6 +758,9 @@ function createPlaneFromStore( ): Promise { const limit = resolveFindLimit(params.limit); let localItems: FindItem[] = []; + let localDegraded: DegradeFlag[] | undefined; + let localEvidence: HybridSearchResult["evidence"] | undefined; + if (wantsLocalChannel(params.sources)) { const local = await store.find({ tenantId: params.tenantId, @@ -742,6 +768,10 @@ function createPlaneFromStore( query: params.query, limit, includeEvidence: true, + ...(params.kinds !== undefined ? { kinds: params.kinds } : {}), + ...(params.entityIds !== undefined + ? { entityIds: params.entityIds } + : {}), }); localItems = local.items.map((it) => ({ documentId: it.documentId, @@ -750,7 +780,10 @@ function createPlaneFromStore( score: it.score, kind: it.kind, citation: it.citation, + ...(it.updatedAt !== undefined ? { updatedAt: it.updatedAt } : {}), })); + localDegraded = local.degraded as DegradeFlag[] | undefined; + localEvidence = local.evidence; } const live = await collectLiveItems({ @@ -762,8 +795,26 @@ function createPlaneFromStore( filter: params.sources, }); + // No live channel activity → preserve store evidence semantics. + if ( + live.items.length === 0 && + live.degraded.length === 0 && + (options.sources ?? []).length === 0 + ) { + if (params.includeEvidence) { + return { + items: localItems, + evidence: localEvidence ?? "none", + ...(localDegraded ? { degraded: localDegraded } : {}), + }; + } + return { items: localItems }; + } + return mergeToFindResult({ localItems, + ...(localDegraded !== undefined ? { localDegraded } : {}), + ...(localEvidence !== undefined ? { localEvidence } : {}), liveItems: live.items, liveDegraded: live.degraded, limit, @@ -780,6 +831,12 @@ function createPlaneFromStore( }, async ask(params) { + // Capability layer. Callers reaching the plane in-process bypass the + // HTTP surface's `requireGrant("knowledge", ...)` route guard, so the + // check has to live here — AUTH.md is explicit that the capability and + // data layers are independent and BOTH must allow. Per-document + // visibility (enforced inside the store) is not a substitute for "may + // this principal search at all". if (!grants) { throw new KnowledgeError( 501, @@ -795,6 +852,8 @@ function createPlaneFromStore( "find", grants.conditionRegistry, ); + // `effect: null` means no grant matched at all — deny by default, same + // as an explicit deny. Only an explicit allow proceeds. if (decision.effect !== "allow") { const effect = decision.effect ?? "no-matching-grant"; log.info( @@ -806,6 +865,8 @@ function createPlaneFromStore( ); throw new KnowledgeNotPermittedError(); } + // Fail closed on missing generate *before* retrieval so a misconfigured + // host gets the promised 501 instead of paying for search. if (!options.generate) { throw new KnowledgeError( 501, @@ -884,19 +945,23 @@ function createPlaneFromStore( const { visibility, blockPrincipalIds } = resolveShareAndVisibility(params); + const externalRef = + params.externalRef ?? + `knowledge:${params.tenantId}:${crypto.randomUUID()}`; + return store.add({ tenantId: params.tenantId, principalId: params.principalId, title, text, visibility, + externalRef, ...(blockPrincipalIds !== undefined ? { blockPrincipalIds } : {}), ...(params.attributes !== undefined ? { attributes: params.attributes } : {}), - ...(params.externalRef !== undefined - ? { externalRef: params.externalRef } - : {}), + ...(params.adapter !== undefined ? { adapter: params.adapter } : {}), + ...(params.kind !== undefined ? { kind: params.kind } : {}), }); }, @@ -921,15 +986,11 @@ function createPlaneFromStore( } /** - * Default plane: engine pgvector store + hybrid search. + * Default DocumentStore: engine pgvector + hybrid search + timeline. + * Owns construction-time rerank validation, FTS verification, and ACL + * block-list post-filter. The plane never opens Postgres itself. */ -function createPlaneFromEngine( - config: KnowledgeConfig, - grants: GrantConfig | undefined, - options: KnowledgePlaneOptions, -): KnowledgePlane { - const memoryApi = makeRememberRecall(options); - +function createEngineDocumentStore(config: KnowledgeConfig): DocumentStore { // Catch a chunk-size / reranker-limit mismatch at construction time, rather // than silently on every find once the reranker starts rejecting batches. // Throws instead of warning: a mismatch means every rerank call for this @@ -972,8 +1033,8 @@ function createPlaneFromEngine( ); /** - * Hybrid retrieval + block-list post-filter. Shared by find() and ask(). - * Returns the full HybridSearchResult so ask can synthesize from hits. + * Hybrid retrieval + block-list post-filter. + * Returns the full HybridSearchResult so evidence/degrade pass through. */ async function retrieve(params: { tenantId: string; @@ -1046,207 +1107,19 @@ function createPlaneFromEngine( } } - const plane: KnowledgePlane = { - async find(params) { - const limit = resolveFindLimit(params.limit); - let localItems: FindItem[] = []; - let localDegraded: DegradeFlag[] | undefined; - let localEvidence: HybridSearchResult["evidence"] | undefined; - - if (wantsLocalChannel(params.sources)) { - const result = await retrieve({ - tenantId: params.tenantId, - principalId: params.principalId, - query: params.query, - ...(limit !== undefined ? { k: limit } : {}), - ...(params.kinds !== undefined ? { kinds: params.kinds } : {}), - ...(params.entityIds !== undefined - ? { entityIds: params.entityIds } - : {}), - }); - localItems = hitsToFindItems(result.hits); - localDegraded = result.degraded; - localEvidence = result.evidence; - } - - const live = await collectLiveItems({ - sources: options.sources, - query: params.query, - tenantId: params.tenantId, - principalId: params.principalId, - limit, - filter: params.sources, - }); - - // No live channel activity → preserve hybrid evidence semantics. - if ( - live.items.length === 0 && - live.degraded.length === 0 && - (options.sources ?? []).length === 0 - ) { - if (params.includeEvidence) { - return { - items: localItems, - evidence: localEvidence ?? "none", - ...(localDegraded ? { degraded: localDegraded } : {}), - }; - } - return { items: localItems }; - } - - return mergeToFindResult({ - localItems, - ...(localDegraded !== undefined ? { localDegraded } : {}), - liveItems: live.items, - liveDegraded: live.degraded, - limit, - ...(params.sources !== undefined ? { sources: params.sources } : {}), - ...(params.includeEvidence !== undefined - ? { includeEvidence: params.includeEvidence } - : {}), - }); - }, - - async ask(params) { - // Capability layer. Callers reaching the plane in-process bypass the - // HTTP surface's `requireGrant("knowledge", ...)` route guard, so the - // check has to live here — AUTH.md is explicit that the capability and - // data layers are independent and BOTH must allow. Per-document - // visibility (enforced inside `find`) is not a substitute for "may - // this principal search at all". - // Same action as HTTP find/ask/recent: knowledge:find. - if (!grants) { - throw new KnowledgeError( - 501, - "ask() requires a GrantConfig. Pass grants to " + - "createKnowledgePlane/mountKnowledgeEngine.", - ); - } - const decision = await authorize( - grants.grantStore, - params.principalId, - params.tenantId, - "knowledge", - "find", - grants.conditionRegistry, - ); - // `effect: null` means no grant matched at all — deny by default, same - // as an explicit deny. Only an explicit allow proceeds. - if (decision.effect !== "allow") { - // Interpolate into the message string: some sinks only render the - // template, not the structured context object (see src/log.ts). - const effect = decision.effect ?? "no-matching-grant"; - log.info( - `ask: denied knowledge:find for ${params.principalId} (effect=${effect})`, - { - principalId: params.principalId, - effect, - }, - ); - throw new KnowledgeNotPermittedError(); - } - - // Fail closed on missing generate *before* retrieval so a misconfigured - // host gets the promised 501 instead of paying for hybrid search (or - // surfacing a DB error that masks the real problem). - if (!options.generate) { - throw new KnowledgeError( - 501, - "ask() requires a `generate` function. Pass one to " + - "createKnowledgePlane/mountKnowledgeEngine, wired to your " + - "inference layer.", - ); - } - - // Find AS the asking principal — the per-document ACL boundary, - // including the block-list post-filter. includeEvidence so synthesis - // can report evidence; goes through plane.find so tests can stub it. - const findResult = await plane.find({ - tenantId: params.tenantId, - principalId: params.principalId, - query: params.query, - includeEvidence: true, - ...(params.limit !== undefined ? { limit: params.limit } : {}), - ...(params.sources !== undefined ? { sources: params.sources } : {}), - }); - - const mem = await recallForAsk({ - memory: options.memory, - includeMemory: params.includeMemory, - tenantId: params.tenantId, - principalId: params.principalId, - query: params.query, - }); - - const answer = await synthesizeAnswer( - params.query, - { - hits: findItemsToHits(findResult.items), - evidence: findResult.evidence ?? "none", - }, - options.generate, - mem.texts, - ); - const degraded: DegradeFlag[] = [ - ...(findResult.degraded ?? []), - ...mem.degraded, - ]; - return { - ...answer, - ...(degraded.length > 0 ? { degraded } : {}), - }; - }, - + return { async add(params) { await ensureVerified(); - const hasContent = params.content !== undefined; - const hasFile = params.file !== undefined; - if (hasContent === hasFile) { - throw new KnowledgeError( - 400, - "provide exactly one of content or file", - ); - } - - let title: string; - let text: string; - if (params.content) { - title = params.content.title; - text = params.content.text; - } else { - const file = params.file!; - if (!options.textExtractor) { - throw new KnowledgeError( - 400, - "file requires a textExtractor on the knowledge plane", - ); - } - const extracted = await options.textExtractor.extract({ - bytes: file.bytes, - ...(file.mimeType !== undefined ? { mimeType: file.mimeType } : {}), - ...(file.filename !== undefined ? { filename: file.filename } : {}), - }); - text = extracted.text; - title = - file.title ?? - extracted.title ?? - file.filename ?? - "untitled"; - } - - const { visibility, blockPrincipalIds } = - resolveShareAndVisibility(params); - - const adapter = params.adapter ?? "mcp"; + const adapter = params.adapter ?? "http"; const externalRef = params.externalRef ?? `knowledge:${params.tenantId}:${crypto.randomUUID()}`; const attributes: Record = { ...(params.attributes ?? {}), }; - if (blockPrincipalIds && blockPrincipalIds.length > 0) { - attributes["acl_block"] = JSON.stringify(blockPrincipalIds); + if (params.blockPrincipalIds && params.blockPrincipalIds.length > 0) { + attributes["acl_block"] = JSON.stringify(params.blockPrincipalIds); } const captureResult = await captureDocument(deps, { @@ -1255,37 +1128,55 @@ function createPlaneFromEngine( occurredAt: new Date().toISOString(), document: { kind: params.kind ?? "note", - title, + title: params.title, externalRef, - visibility, + visibility: params.visibility, entityHints: [], - chunks: [{ ordinal: 0, text }], + chunks: [{ ordinal: 0, text: params.text }], actor: { kind: "human", principalId: params.principalId }, contentHash: "", // recomputed canonically in adapt-and-plan ...(Object.keys(attributes).length > 0 ? { attributes } : {}), }, }); - // Both captured and noop return documentId — always surface it. return { documentId: captureResult.documentId }; }, + async find(params) { + const result = await retrieve({ + tenantId: params.tenantId, + principalId: params.principalId, + query: params.query, + ...(params.limit !== undefined ? { k: params.limit } : {}), + ...(params.kinds !== undefined ? { kinds: params.kinds } : {}), + ...(params.entityIds !== undefined + ? { entityIds: params.entityIds } + : {}), + }); + const items = hitsToFindItems(result.hits); + if (params.includeEvidence) { + return { + items, + evidence: result.evidence, + ...(result.degraded ? { degraded: result.degraded } : {}), + }; + } + return { + items, + ...(result.degraded ? { degraded: result.degraded } : {}), + }; + }, + async recent(params) { - const limit = resolveRecentLimit(params.limit); return listTimelineEvents({ db, tenantId: params.tenantId, principalId: params.principalId, - ...(limit !== undefined ? { limit } : {}), + ...(params.limit !== undefined ? { limit: params.limit } : {}), }); }, - remember: memoryApi.remember, - recall: memoryApi.recall, - async close() { await sql.end({ timeout: 5 }); }, }; - - return plane; } diff --git a/src/ports/fakes.ts b/src/ports/fakes.ts index 5518cc4..f5f2e70 100644 --- a/src/ports/fakes.ts +++ b/src/ports/fakes.ts @@ -155,12 +155,18 @@ export function createFakeSourceProvider( const q = params.query.toLowerCase(); const limit = params.limit ?? 8; return catalog - .filter( - (item) => - item.adapter === id && - (item.title.toLowerCase().includes(q) || - item.snippet.toLowerCase().includes(q)), - ) + .filter((item) => { + if (item.adapter !== id) return false; + // Fail-closed: seed rows may carry tenantId for tenancy tests. + const row = item as LiveSearchItem & { tenantId?: string }; + if (row.tenantId !== undefined && row.tenantId !== params.tenantId) { + return false; + } + return ( + item.title.toLowerCase().includes(q) || + item.snippet.toLowerCase().includes(q) + ); + }) .slice(0, limit); }, }; diff --git a/src/ports/types.ts b/src/ports/types.ts index 8948aff..37f9087 100644 --- a/src/ports/types.ts +++ b/src/ports/types.ts @@ -1,8 +1,11 @@ /** - * Port contracts for pluggable storage, live sources, and personal memory. + * Port contracts for pluggable storage, live sources, and optional personal memory. * - * DocumentStore + SourceProvider are the M2 foundation. MemoryProvider is - * wired into ask (includeMemory) and plane.remember/recall in M3. + * DocumentStore is the durable backend for add/find/recent (default: local + * pgvector). Hosts replace it with Mem0, Supermemory, or fakes — no dual store. + * SourceProvider is tools-shaped live connectors (e.g. Linear), not a store. + * MemoryProvider is an optional ask side-channel (includeMemory); not how you + * swap backends. */ import type { VisibilitySpec } from "../core/schemas/document.ts"; import type { @@ -22,6 +25,10 @@ export type DocumentStoreAddParams = { blockPrincipalIds?: string[]; attributes?: Record; externalRef?: string; + /** Capture adapter id (default engine store uses `"http"`). */ + adapter?: string; + /** Document kind (default engine store uses `"note"`). */ + kind?: string; }; export type DocumentStoreFindParams = { @@ -30,6 +37,10 @@ export type DocumentStoreFindParams = { query: string; limit?: number; includeEvidence?: boolean; + /** Narrow local retrieval by document kind (unset/`[]` = no filter). */ + kinds?: string[]; + /** Narrow local retrieval by linked entity ids (unset/`[]` = no filter). */ + entityIds?: string[]; }; export type DocumentStoreFindItem = { @@ -66,8 +77,10 @@ export type DocumentStoreRecentEvent = { }; /** - * Durable local document plane. Default implementation is the engine's - * pgvector store; hosts may inject a fake or alternate backend. + * Durable document plane. Default implementation is the engine's pgvector + * store. Hosts inject Mem0 / Supermemory / fakes via `options.documentStore` + * to replace local Postgres entirely — this is the only product path for + * swapping backends. */ export type DocumentStore = { add(params: DocumentStoreAddParams): Promise<{ documentId: string }>; @@ -80,7 +93,7 @@ export type DocumentStore = { /** * One live hit from a SourceProvider.searchLive call. - * Dedupe key for merge is `adapter:externalRef` (CL-5227). + * Dedupe key for merge is `adapter:externalRef`. */ export type LiveSearchItem = { adapter: string; @@ -110,9 +123,9 @@ export type SourceProvider = { }; /** - * Personal memory port. Adapters implement this in packages/*; - * core never imports vendor SDKs. Writes are host-owned (remember); - * ask only recalls when includeMemory is true. + * Optional personal-memory side channel for ask(includeMemory). Not a + * DocumentStore replacement — Mem0/Supermemory product adapters implement + * DocumentStore, not this port. */ export type MemoryProvider = { remember(params: { diff --git a/src/routes/add.ts b/src/routes/add.ts index 3863669..a64525d 100644 --- a/src/routes/add.ts +++ b/src/routes/add.ts @@ -5,6 +5,7 @@ import { type } from "arktype"; import { formatCaughtError, log } from "../log.ts"; import { parseAcl } from "../acl.ts"; +import { KnowledgeError } from "../knowledge.ts"; import type { RouteDeps } from "./deps.ts"; import { caller, grantGuard, requirePrincipal } from "./deps.ts"; @@ -34,6 +35,7 @@ export function mountAddRoute(app: Hono, deps: RouteDeps): void { 400: { description: "Invalid request or ACL" }, 401: { description: "No principal on the request context" }, 403: { description: "Missing the knowledge:add grant" }, + 502: { description: "add failed" }, }, }), requirePrincipal(), @@ -56,6 +58,12 @@ export function mountAddRoute(app: Hono, deps: RouteDeps): void { }); return c.json({ documentId }); } catch (err) { + if (err instanceof KnowledgeError) { + return c.json( + { error: err.message }, + err.status as 400 | 501, + ); + } const errMessage = formatCaughtError(err); log.error(`knowledge add failed: ${errMessage}`, { error: errMessage }); return c.json({ error: "add failed" }, 502); diff --git a/src/routes/ask.ts b/src/routes/ask.ts index 23e3879..36d6017 100644 --- a/src/routes/ask.ts +++ b/src/routes/ask.ts @@ -14,6 +14,8 @@ import { caller, grantGuard, requirePrincipal } from "./deps.ts"; const AskRequest = type({ query: "string >= 1", "limit?": "1 <= number.integer <= 50", + "sources?": "string[]", + "includeMemory?": "boolean", }); const AskResponse = type({ @@ -25,6 +27,7 @@ const AskResponse = type({ citation: "unknown", }).array(), evidence: "'strong'|'weak'|'none'", + "degraded?": "string[]", }); export function mountAskRoute(app: Hono, deps: RouteDeps): void { @@ -52,7 +55,7 @@ export function mountAskRoute(app: Hono, deps: RouteDeps): void { grantGuard(deps, "find"), validator("json", AskRequest), async (c) => { - const { query, limit } = c.req.valid("json"); + const { query, limit, sources, includeMemory } = c.req.valid("json"); const { scopeId, subjectId } = caller(c); try { const result = await deps.knowledge.ask({ @@ -60,6 +63,8 @@ export function mountAskRoute(app: Hono, deps: RouteDeps): void { tenantId: scopeId, principalId: subjectId, ...(limit !== undefined ? { limit } : {}), + ...(sources !== undefined ? { sources } : {}), + ...(includeMemory !== undefined ? { includeMemory } : {}), }); return c.json(result); } catch (err) { diff --git a/src/routes/find.ts b/src/routes/find.ts index fc269a6..c8ec46f 100644 --- a/src/routes/find.ts +++ b/src/routes/find.ts @@ -20,9 +20,10 @@ const FindRequest = type({ "limit?": "1 <= number.integer <= 50", "kinds?": "string[]", "entity_ids?": "string[]", + "sources?": "string[]", + "includeEvidence?": "boolean", }); -// includeEvidence is always true on HTTP so the wire reports evidence. const FindResponse = type({ items: type({ documentId: "string", @@ -45,7 +46,8 @@ export function mountFindRoute(app: Hono, deps: RouteDeps): void { description: "`kinds`/`entity_ids` scope every retrieval channel (lexical and " + "dense) before results are fused, so every hit matches the " + - "requested kind/entity.", + "requested kind/entity. `sources` optionally restricts local + live " + + "channels; `includeEvidence` defaults true on the wire.", responses: { 200: { description: "Ranked items with evidence", @@ -56,23 +58,28 @@ export function mountFindRoute(app: Hono, deps: RouteDeps): void { 400: { description: "Invalid query" }, 401: { description: "No principal on the request context" }, 403: { description: "Missing the knowledge:find grant" }, + 502: { description: "find failed" }, }, }), requirePrincipal(), grantGuard(deps, "find"), validator("json", FindRequest), async (c) => { - const { query, limit, kinds, entity_ids } = c.req.valid("json"); + const { query, limit, kinds, entity_ids, sources, includeEvidence } = + c.req.valid("json"); const { scopeId, subjectId } = caller(c); try { const result = await deps.knowledge.find({ query, tenantId: scopeId, principalId: subjectId, - includeEvidence: true, + // Default true on HTTP so the wire always reports evidence unless + // the client explicitly opts out. + includeEvidence: includeEvidence ?? true, ...(limit !== undefined ? { limit } : {}), ...(kinds !== undefined ? { kinds } : {}), ...(entity_ids !== undefined ? { entityIds: entity_ids } : {}), + ...(sources !== undefined ? { sources } : {}), }); return c.json(result); } catch (err) { diff --git a/src/routes/recent.ts b/src/routes/recent.ts index e71e186..9ff01d5 100644 --- a/src/routes/recent.ts +++ b/src/routes/recent.ts @@ -1,12 +1,21 @@ import type { Hono } from "hono"; import type { TenantEnv } from "@intx/hub-api"; -import { describeRoute, resolver } from "hono-openapi"; +import { describeRoute, resolver, validator } from "hono-openapi"; import { type } from "arktype"; import { formatCaughtError, log } from "../log.ts"; +import { + KnowledgeError, + RECENT_LIMIT_MAX, + RECENT_LIMIT_MIN, +} from "../knowledge.ts"; import type { RouteDeps } from "./deps.ts"; import { caller, grantGuard, requirePrincipal } from "./deps.ts"; +const RecentQuery = type({ + "limit?": "string", +}); + const RecentResponse = type({ events: type({ at: "string", @@ -17,6 +26,19 @@ const RecentResponse = type({ }).array(), }); +function parseLimit(raw: string | undefined): number | undefined { + if (raw === undefined || raw === "") return undefined; + const n = Number(raw); + if ( + !Number.isInteger(n) || + n < RECENT_LIMIT_MIN || + n > RECENT_LIMIT_MAX + ) { + return undefined; + } + return n; +} + export function mountRecentRoute(app: Hono, deps: RouteDeps): void { app.get( "/api/knowledge/recent", @@ -30,6 +52,7 @@ export function mountRecentRoute(app: Hono, deps: RouteDeps): void { "application/json": { schema: resolver(RecentResponse) }, }, }, + 400: { description: "Invalid limit query param" }, 401: { description: "No principal on the request context" }, 403: { description: "Missing the knowledge:find grant" }, 502: { description: "Recent query failed" }, @@ -37,15 +60,34 @@ export function mountRecentRoute(app: Hono, deps: RouteDeps): void { }), requirePrincipal(), grantGuard(deps, "find"), + validator("query", RecentQuery), async (c) => { const { scopeId, subjectId } = caller(c); + const rawLimit = c.req.valid("query").limit; + if ( + rawLimit !== undefined && + rawLimit !== "" && + parseLimit(rawLimit) === undefined + ) { + return c.json( + { + error: `limit must be an integer from ${RECENT_LIMIT_MIN} to ${RECENT_LIMIT_MAX}`, + }, + 400, + ); + } + const limit = parseLimit(rawLimit); try { const events = await deps.knowledge.recent({ tenantId: scopeId, principalId: subjectId, + ...(limit !== undefined ? { limit } : {}), }); return c.json({ events }); } catch (err) { + if (err instanceof KnowledgeError) { + return c.json({ error: err.message }, err.status as 400); + } const errMessage = formatCaughtError(err); log.error(`knowledge recent failed: ${errMessage}`, { error: errMessage, diff --git a/src/routes/routes.test.ts b/src/routes/routes.test.ts index 5361db9..8334fe2 100644 --- a/src/routes/routes.test.ts +++ b/src/routes/routes.test.ts @@ -39,12 +39,11 @@ function stubPlane(opts?: { askImpl?: KnowledgePlane["ask"]; }) { const added: { title: string; tenantId: string; principalId: string }[] = []; - const searched: Array< - Pick< - Parameters[0], - "kinds" | "entityIds" | "limit" - > - > = []; + const searched: Array<{ + kinds: string[] | undefined; + entityIds: string[] | undefined; + limit: number | undefined; + }> = []; const catalog = opts?.timelineCatalog ?? []; const plane: KnowledgePlane = { find: async (p) => { diff --git a/src/services/search.test.ts b/src/services/search.test.ts index 958475a..26a46cf 100644 --- a/src/services/search.test.ts +++ b/src/services/search.test.ts @@ -469,7 +469,12 @@ describe("fetchDenseCandidates kind/entity filtering", () => { const rawSql = { unsafe: (sqlText: string) => Promise.resolve( - sqlText.includes("FROM knowledge_embed_model") ? [MODEL_ROW] : [], + // CL-5233 qualified the table; keep the pre-qualify form so an + // accidental revert still fails this suite the same way. + sqlText.includes('FROM "knowledge"."embed_model"') || + sqlText.includes("FROM knowledge_embed_model") + ? [MODEL_ROW] + : [], ), begin: (cb: (t: FakeTx) => Promise) => cb(tx), }; diff --git a/src/services/timeline.ts b/src/services/timeline.ts index 5edca0e..3c73567 100644 --- a/src/services/timeline.ts +++ b/src/services/timeline.ts @@ -126,7 +126,7 @@ export function filterTimelineRowsForPrincipal( * (the same helper search uses through `blockedDocumentIds`). * * One row per document (active live version), ordered by last_seen_at DESC. - * Wire `source` is the document adapter (HTTP capture defaults to "mcp"). + * Wire `source` is the document adapter (HTTP add defaults to "http"). * Wire `principalId` is knowledge_version.created_by_principal_id. */ export async function listTimelineEvents(