diff --git a/.corbits/review-round2-bruckheimer.md b/.corbits/review-round2-bruckheimer.md new file mode 100644 index 0000000..1e7e6e0 --- /dev/null +++ b/.corbits/review-round2-bruckheimer.md @@ -0,0 +1,41 @@ +# Bruckheimer — PR #31 round 2 (HEAD `6189b56`) + +## One-liner + +Foundation so a company-brain distiller can write **inferred claims** with +provenance, rank them in time, re-distill offline without trashing live search, +and share docs with real grants — not just tags. + +## Hook progress + +| Piece | Status for the hook | +|-------|---------------------| +| Claim-bearing / derived_from / provenance | Shipped | +| Temporal classes + validity | Shipped | +| Staged transform / promote / demote | Shipped (code); weak tests | +| Share grants materialization | Shipped (fail-soft without writable store) | +| Capture feed / distiller workflow | **Not this PR** (CL-5868/5869) | +| Schema/config host-friendly | DATABASE_URL + memory schema — good | + +## Product blockers + +1. **Dense entity filter broken after rename** — if any host/search UI filters by + entity, dense path dies. Fix before merge. +2. **Promote rollback untested** — if demote is the safety valve for bad + distillation, untested demote is a product risk, not just eng debt. +3. **Share without WritableGrantStore** only warns — hosts will think share + “worked.” Consider fail-loud option or return receipt with + `grantsMaterialized: false` (follow-up OK). + +## Breaking-change cost/benefit + +- `knowledge` → `memory` schema: right name for the product; cost is fresh + install only. Acceptable if no production tenants on old schema. +- `DATABASE_URL` preferred: lowers host friction (one DB). Risk: host with two + URLs silently picks the wrong one — document clearly (done in mount-config). + +## Ship advice + +Fix the dense SQL bug, clarify CHANGELOG on versionId, merge as foundation. +Do not hold the PR for the distiller itself. Next product milestone is feed + +workflow, not more schema polish. diff --git a/.corbits/review-round2-convergence.md b/.corbits/review-round2-convergence.md new file mode 100644 index 0000000..dc84bb3 --- /dev/null +++ b/.corbits/review-round2-convergence.md @@ -0,0 +1,88 @@ +# Convergence review — PR #31 round 2 (HEAD `6189b56`) + +**Method:** 8 deep lenses in-session (fleet spawn blocked: Codex profile +`fleur` unauthorized). Lenses: critique, greybeard, gaasbot, neckbeard, +bruckheimer, security, schema, OSS. Artifacts: +`.corbits/review-round2-*.md`. + +## Converged verdict + +**CHANGES REQUESTED — one hard blocker, then merge-eligible.** + +All eight lenses agree the distillation foundation is the right shape and that +prior promote/demote/versionId work improved the bar. All eight that touched +search/schema flag the same ship-blocker. + +--- + +## Must-fix before merge (unanimous / multi-lens) + +| ID | Finding | Lenses | Action | +|----|---------|--------|--------| +| **M1** | Dense entity filter SQL still uses `knowledge_edge`; table is `"memory"."edge"` (`src/services/search.ts:537`) | critique, greybeard, gaasbot, neckbeard, security, schema, OSS, bruckheimer | Fix SQL + add regression test (dense path with entityIds) | +| **M2** | CHANGELOG Unreleased “Previously” claims add returns only `{ documentId }`; code returns `versionId` | critique, gaasbot, greybeard, OSS | Correct CHANGELOG | + +## Should-fix (same PR if cheap; else ticket) + +| ID | Finding | Lenses | Action | +|----|---------|--------|--------| +| **S1** | No promote/demote service E2E regression | critique, greybeard, gaasbot, bruckheimer | Add transform test: ensure→promote→demote restores model_key + generation | +| **S2** | search.test mocks still accept `FROM knowledge_embed_model` | neckbeard, OSS | Match only `"memory"."embed_model"` | +| **S3** | Document that transform/promote/demote are host-privileged (no principal on API, no HTTP) | security, gaasbot, greybeard | Short IMPLEMENTATION / AUTHZ note | + +## Follow-up (do not block merge) + +| ID | Finding | Lenses | +|----|---------|--------| +| F1 | `setActiveEmbedModelExclusive` two-step race | critique, security | +| F2 | Promote/demote multi-step windows; consider tenant advisory lock | critique, greybeard, security | +| F3 | Share fail-soft without WritableGrantStore / appendAccessTags — receipt field | critique, bruckheimer, security | +| F4 | TS `knowledge*` identifiers + migration filenames + comment rot | neckbeard, schema | +| F5 | Ops note for `ALTER SCHEMA knowledge RENAME TO memory` if any old install | greybeard, schema | +| F6 | README advanced surface (transform exports) | OSS, bruckheimer | + +## Explicit non-goals this PR + +- Distiller workflow / capture feed (CL-5868/5869) +- HTTP routes for transform +- Bulk rename of knowledge* TypeScript symbols +- Re-introducing separate knowledge DB + +## Steelman of “merge as-is” + +Tests are green; entityIds on dense may be rare; rename is fresh-install-only. +**Rejected:** a single untested raw-SQL island after a schema rename is exactly +the class of bug that survives CI and fails first real use. Fix is trivial. + +## Steelman of “hold for promote E2E” + +Demote is the safety valve for bad distillation. **Partial accept:** S1 is +high value but not a correctness hole in the current code path (demote restore +exists). Prefer same-PR if <1h; else ticket linked from PR. + +## Converged fix order + +1. **M1** fix + test +2. **M2** CHANGELOG +3. **S2** tighten mocks (with M1 test) +4. **S1** if time +5. **S3** one paragraph +6. Re-run typecheck + test → push + +## Bar after fixes + +| Bar | Status after M1+M2 | +|-----|--------------------| +| Security | Pass (with S3 doc preferred) | +| Product | Pass foundation | +| Greybeard / architecture | Pass | +| OSS quality | Pass pre-1.0 | +| Critique | Pass with S1 follow-up | + +--- + +## Note on fleet + +All 8 `task` spawns failed: `Codex profile "fleur" is not authorized`. Reviews +were executed in-session with the same multi-lens briefs. Re-auth `/model` +(profile fleur) to restore sub-agent fleet for future rounds. diff --git a/.corbits/review-round2-critique.md b/.corbits/review-round2-critique.md new file mode 100644 index 0000000..7e4ae77 --- /dev/null +++ b/.corbits/review-round2-critique.md @@ -0,0 +1,74 @@ +# Critique — PR #31 round 2 (HEAD `6189b56`) + +In-session (fleet blocked: Codex profile `fleur`). Diff `origin/main..HEAD`. +Typecheck clean; 367 tests green at last gate. + +## Verdict + +**CHANGES REQUESTED.** Prior promote/demote and versionId fixes landed, but the +`knowledge` → `memory` rename left a **live dense-path SQL bug**, and +promote/demote still lack end-to-end regression tests. + +## Critical + +1. **Dense entity filter still queries `knowledge_edge`** + (`src/services/search.ts:537`). After schema rename, the table is + `"memory"."edge"`. Any dense search with `entityIds` will fail at runtime + (`relation "knowledge_edge" does not exist`). Lexical path is fine (Drizzle + `knowledgeEdge` → `memory.edge`). No test exercises dense+entityIds. + +## High + +2. **No automated promote → demote → dense restore E2E.** Registry unit tests + cover ensure/activate/byKey, but nothing asserts: staged run under model B + does not flip live active; promote swaps generation + activates staged; + demote restores generation **and** prior model_key. CL-5872 rollback + acceptance still untested at the service layer. + +3. **Promote/demote are multi-step outside a single transaction.** Activate + dense, then version swap (or reverse on demote). Concurrent promote of two + generations, or crash mid-window, can leave dense target and generation tags + briefly inconsistent. Documented preference is intentional; still a + production footgun without locks / single-flight. + +## Medium + +4. **`setActiveEmbedModelExclusive` is two non-atomic UPDATEs** + (`embed-model-registry.ts:309-325`). Concurrent activate of A and B can + leave two `active` rows until next exclusive call; `ORDER BY updated_at` + picks one, but window exists. + +5. **Transform plane methods take only `tenantId` / `configId` — no principal.** + In-process API; no HTTP routes. Correct for library shape, but any host that + re-exports without its own grant check hands promote/demote to any caller + who can reach the plane. Docs should state “host must authorize.” + +6. **Share path still fail-soft** when `appendAccessTags` or WritableGrantStore + missing (`memory.ts:707-727`): warns, continues, peers fail-closed. Easy to + miss in production. + +7. **CHANGELOG drift:** Unreleased “Previously” still says `add` returns + `{ documentId }` only; wire now returns `{ documentId, versionId }`. + +## Low / Nits + +8. Comments and enums still say `knowledge.version` / `knowledge.embed_model` + (`enums.ts`, `embed-model-registry.ts:34`, `generation.ts`). +9. Migration file still named `0002_knowledge_baseline.sql` while creating + `memory.*`. +10. TS exports still `knowledgeDocument` / `knowledgeVersion` under memory schema. + +## Test gaps + +- Dense search + `entityIds` (would catch Critical #1). +- `promoteGeneration` / `demoteGeneration` service tests with fake SQL + version rows. +- Concurrent exclusive activate (optional stress). +- `loadMemoryConfig` DATABASE_URL vs KNOWLEDGE preference already covered. + +## Assumptions challenged + +- “Rename was complete because migrations and drizzle use memory” — raw SQL + island in dense path was not. +- “367 green ⇒ rename safe” — unit tests mock embed_model with dual match + (`knowledge_embed_model` OR `memory.embed_model`) and never hit entity filter + raw SQL. diff --git a/.corbits/review-round2-gaasbot.md b/.corbits/review-round2-gaasbot.md new file mode 100644 index 0000000..c1d1735 --- /dev/null +++ b/.corbits/review-round2-gaasbot.md @@ -0,0 +1,44 @@ +# Gaasbot (CTO) — PR #31 round 2 (HEAD `6189b56`) + +## CTO verdict + +**Right foundation, one ship-blocker.** This is the correct shape for +resident distillation: claim-bearing + temporal + staged transform + grants. +Do not expand scope into the distiller workflow (CL-5869) on this PR. + +## Must-fix-before-merge + +1. Fix dense-path `knowledge_edge` → `"memory"."edge"` (`search.ts:537`). +2. Add a regression test that would fail on that bug (dense fetch with + entityIds, assert SQL contains `"memory"."edge"` or run against real SQL + mock that only knows memory.edge). +3. Fix CHANGELOG: add returns `versionId`; remove contradictory “Previously” + line or mark superseded. + +## Can-ship-with-followups + +- Promote/demote service-level tests. +- Tenant-scoped advisory lock on promote/demote. +- Atomic exclusive activate (single SQL CTE or transaction). +- Explicit “host authorizes transform APIs” note in IMPLEMENTATION.md. +- Rename TS `knowledge*` symbols in a dedicated PR (not this one). + +## Defer + +- HTTP routes for transform/promote (in-process is fine for v1 distiller). +- Capture feed (CL-5868), relevancy (CL-5867), retention (CL-5871). +- Upgrade migration from `knowledge` schema for old DBs unless a customer exists. + +## Architecture notes + +- Exporting transform + embed registry from package root is aggressive but OK + for the distiller as first-party consumer. Keep them off HTTP until grants + exist. +- Preferring `DATABASE_URL` is correct for “same Postgres, own schema.” Warn + hosts that still set both URLs with different values — preferred wins. +- Do not re-introduce a separate knowledge DB requirement. + +## Priority + +Blocker fix is a one-liner + test. Merge after that; iterate on promote +hardening in the same branch if cheap, else follow-up ticket. diff --git a/.corbits/review-round2-greybeard.md b/.corbits/review-round2-greybeard.md new file mode 100644 index 0000000..7af7025 --- /dev/null +++ b/.corbits/review-round2-greybeard.md @@ -0,0 +1,59 @@ +# Greybeard — PR #31 round 2 (HEAD `6189b56`) + +## Verdict + +**HOLD for one correctness fix; then ship foundation.** Architecture of +ensure-vs-activate, claim-bearing, temporal classes, and share materialization +is sound. Schema rename + DATABASE_URL is the right long-term shape. + +## Ship / hold + +**Hold** until dense `entityIds` SQL is fixed (`knowledge_edge` → `"memory"."edge"`). +After that: **ship with follow-ups** (promote E2E tests, exclusive activate +transaction, host authz docs for transform). + +## Critical / High + +1. **Raw SQL residue after schema rename** — `search.ts:537` `knowledge_edge`. + Irreversible-looking renames that leave one path broken are worse than no + rename: green CI + red prod. + +2. **Fresh-install-only schema rename** is honest in CHANGELOG but operationally + harsh. Acceptable for pre-1.0 / no prod tenants; document a one-shot + `ALTER SCHEMA knowledge RENAME TO memory` + table renames for anyone who + already migrated under `knowledge`. + +## Medium — design debt (acceptable for now) + +3. **JS identifiers lag schema** (`knowledgeDocument` table → `document`). Fine + if intentional transitional; pick a rename PR later — do not half-rename. +4. **Embedding tables keyed only by model_key**, multi-tenant rows inside. + Tenant filter on every dense query is load-bearing; keep that invariant in + review checklist forever. +5. **Promote activate-then-swap** preference is documented and reasonable. + Prefer advisory lock per tenant around promote/demote before multi-tenant + production load. +6. **Docs generally lockstep** with DATABASE_URL / memory schema after last + commit; IMPLEMENTATION table and AGENTS.md match. CHANGELOG “Previously” + still contradicts versionId on add. + +## Doc drift + +| Claim | Reality | +|-------|---------| +| CHANGELOG: add returns `{ documentId }` | Returns `{ documentId, versionId }` | +| Comments: knowledge.embed_model | Table is memory.embed_model | +| open.type "memory" | Correct in search.ts | + +## Design decisions that aged well + +- `ensureEmbedModel` vs `activateEmbedModel` split (replay must not steal live). +- `activateEmbedModelByKey` for demote without re-probe. +- `archived_live_model_key` on transform_run. +- Share grants + pass-through condition registry. +- Enum lockstep test (enums.lockstep.test.ts). + +## Recommendation + +Fix Critical SQL → add dense+entityIds test → optional promote E2E → merge. +Do not block on knowledge* TypeScript renames. diff --git a/.corbits/review-round2-neckbeard.md b/.corbits/review-round2-neckbeard.md new file mode 100644 index 0000000..ff03a4b --- /dev/null +++ b/.corbits/review-round2-neckbeard.md @@ -0,0 +1,32 @@ +# Neckbeard — PR #31 round 2 (HEAD `6189b56`) + +## Real bugs hiding as nits + +1. **`knowledge_edge` in raw SQL** (`search.ts:537`) — not a naming nit. **Bug.** +2. **search.test.ts still accepts `FROM knowledge_embed_model`** as a success + path for mocks (`search.test.ts:237, 412`). Teaches the wrong table name; + should only match `"memory"."embed_model"`. + +## Naming debt inventory (cosmetic unless noted) + +| Residue | Severity | +|---------|----------| +| `knowledgeDocument`, `knowledgeVersion`, `knowledgeChunk`, `knowledgeEdge`, `knowledgeEntity`, `knowledgeEmbedModel` exports | Cosmetic / API-internal | +| `KNOWLEDGE_SCHEMA` deprecated alias | OK transitional | +| `migrations/0002_knowledge_baseline.sql` filename | Cosmetic; content correct | +| Comments `knowledge.version`, `knowledge.embed_model` | Doc rot | +| Id prefixes `kver`, `kdoc` | Cosmetic | +| grant-tags test still uses `knowledge.project:ke` as a free-form tag | Fine (host tags) | +| CHANGELOG “Postgres schema name remains knowledge” removed; good | — | + +## Nits + +- Dual match in tests for old embed_model table should die with the rename. +- `### Previously` in CHANGELOG is nonstandard Keep-a-Changelog structure. +- Package still says “knowledge plane” in a few comments (`config.ts` FTS). +- `openTarget` comment still says “generic knowledge doc” (`search.ts:156`). + +## What not to rewrite + +Do not rename all `knowledge*` TS symbols in this PR. Ship the SQL fix and +stop. A bulk rename PR with codemod is fine later. diff --git a/.corbits/review-round2-oss.md b/.corbits/review-round2-oss.md new file mode 100644 index 0000000..890f2a1 --- /dev/null +++ b/.corbits/review-round2-oss.md @@ -0,0 +1,53 @@ +# OSS / public API quality — PR #31 round 2 (HEAD `6189b56`) + +## Public surface inventory (package root) + +- `createMemory`, `loadMemoryConfig`, `runMemoryMigrations`, `MemoryError` +- Routes: `registerMemoryRoutes` +- Ports: DocumentStore types, fakes, WritableGrantStore +- Share: buildShareGrants, materializeShareGrants, MEMORY_SHARE_* +- Transform: createTransformConfig, runTransform, promote/demoteGeneration, … +- Embed registry: ensure/activate/resolve helpers +- Degrade metrics, FTS helpers + +**Note:** Transform + embed registry on the root export is a large surface for +an “add/search/list” product blurb. Acceptable for distiller-as-consumer; +README should mention advanced APIs. + +## Breaking changes completeness + +| Change | CHANGELOG | Code | +|--------|-----------|------| +| Schema knowledge → memory | Yes | Yes | +| DATABASE_URL preferred | Yes | Yes | +| open.type memory | Yes | Yes | +| add returns versionId | **Stale “Previously” says no** | Yes | +| Claim-bearing / temporal / transform | Partial (IMPLEMENTATION) | Yes | + +## Quality bar + +| Area | Pass? | Notes | +|------|-------|-------| +| arktype at edges | Pass | transform params, raw_capture replay | +| Enum lockstep tests | Pass | enums.lockstep.test.ts | +| Tenant SQL discipline | Pass* | *except broken edge table name | +| Module focus | Pass | services split reasonably | +| Docs match exports | Partial | CHANGELOG versionId; comments knowledge.* | +| Test coverage public contracts | Partial | no promote E2E; dense+entityIds missing | +| Semver honesty | Partial | fix Unreleased Previously section | + +## Must-fix for OSS merge + +1. Dense entity SQL table name. +2. CHANGELOG honesty on `MemoryAddResult` / wire body. +3. Regression test for (1). + +## Nice-to-have before wider publish + +- README section: transform/promote privileged, host-gated. +- Drop dual mock match for `knowledge_embed_model` in tests. +- Do not bulk-rename knowledge* TS identifiers in this PR. + +## Verdict + +**Fail OSS bar until Critical SQL + CHANGELOG; then pass for pre-1.0 foundation.** diff --git a/.corbits/review-round2-schema.md b/.corbits/review-round2-schema.md new file mode 100644 index 0000000..d8bb692 --- /dev/null +++ b/.corbits/review-round2-schema.md @@ -0,0 +1,60 @@ +# Schema / migrations — PR #31 round 2 (HEAD `6189b56`) + +## Summary + +Migrations and Drizzle schema consistently use `"memory".…` with short table +names (`document`, `version`, `chunk`, `edge`, …). Config prefers DATABASE_URL. +One application raw-SQL path still names the pre-rename edge table. + +## Critical + +1. **App SQL vs migration mismatch:** `search.ts:537` uses `knowledge_edge`; + migrations create `"memory"."edge"`. Dense entity filter is broken on any + real Postgres. + +## High + +2. **No upgrade path** from prior `knowledge` schema installs — CHANGELOG says + fresh-only. Correct if intentional; add a short ops note (RENAME SCHEMA + + RENAME tables) if anyone already applied old branch migrations. + +3. **Migration filenames** still `0002_knowledge_baseline.sql` etc. Content is + memory.*; confusing for operators grepping filenames. Optional rename of + files is risky if migration ledger already records names — leave filenames, + fix comments at top of 0002. + +## Medium + +4. **`0001_extensions.sql`** must create schema `memory` before 0002 — verify + CREATE SCHEMA IF NOT EXISTS memory (assumed present; was part of rename). +5. **`archived_live_model_key`** text, no FK to embed_model — intentional + (model row may be demoted/deleted); demote fails closed if key missing. +6. **embed_model ON CONFLICT** updates dims/model_id without status change — + correct for ensure. +7. **CHECK/arktype lockstep** covered by `enums.lockstep.test.ts` — good. +8. **Internal FKs** document ← version ← chunk; edge/entity free of control- + plane FKs — matches AGENTS.md. +9. **DATABASE_URL resolution** order correct; tests cover prefer / fallback / + throw. + +## Low + +10. Index names dropped `knowledge_` prefix — good. +11. Dynamic embedding tables: FK to memory.chunk; tenant_id column; no per- + tenant table isolation (by design). + +## Config + +| Source | Behavior | +|--------|----------| +| `memory.databaseUrl` on config | Programmatic hosts | +| `DATABASE_URL` | Preferred env | +| `KNOWLEDGE_DATABASE_URL` | Deprecated alias | +| Neither | throw | + +## Ranked defects + +1. Critical: raw SQL `knowledge_edge` +2. High: document upgrade path or confirm zero external installs +3. Medium: 0002 file header still says “knowledge plane” +4. Low: TS knowledge* symbols / comment rot diff --git a/.corbits/review-round2-security.md b/.corbits/review-round2-security.md new file mode 100644 index 0000000..7c9124c --- /dev/null +++ b/.corbits/review-round2-security.md @@ -0,0 +1,72 @@ +# Security — PR #31 round 2 (HEAD `6189b56`) + +## Summary + +Tenant isolation on SQL paths is generally solid (tenant_id first; grant post- +filter). No auth in package (by design). One correctness issue can cause +hard-fail (DoS of entity-filtered dense search). Transform APIs are privileged +operations without built-in principal checks. + +## Critical + +None for classic IDOR/authz bypass found in this pass. Closest: + +**C-adjacent:** Dense `entityIds` query references non-existent `knowledge_edge` +(`search.ts:537`) — availability/DoS of that code path, not data leak. + +## High + +1. **Transform promote/demote/run lack principal binding** + (`memory.ts:751-798`, `transform.ts`). Anyone who can call the in-process + plane with a tenantId can rewrite that tenant’s live generation and active + embed model. Mitigation: host-only, no HTTP. **Requirement:** document that + hosts must gate these like admin APIs; never expose unauthenticated. + +2. **Shared embedding physical tables across tenants** (table name = + `embedding_` only). Isolation is `WHERE tenant_id = $1` on every + dense query. A missing tenant predicate on a future query is cross-tenant + vector leak. Current dense SQL includes `e.tenant_id = $1 AND c.tenant_id = $1`. + **Keep as permanent review invariant.** + +## Medium + +3. **Exclusive activate race** — two concurrent activates can briefly leave two + active rows; resolve picks latest updated_at. Unlikely privilege issue; + wrong model for search is integrity issue. + +4. **Share grants use `origin: "system"`** (`share-grants.ts:92`) with + always-true condition evaluator. Correct for not fail-closing, but grants + look “system-minted.” Audit trail relies on `conditions.memoryShare` payload. + Ensure host UIs show that payload. + +5. **Promote activate-before-swap window** — dense points at new model while + live versions still old (or reverse on demote). Transient wrong hits, not + cross-tenant. + +6. **`appendAccessTags` optional** — if missing, peer grants may not match tags; + peers fail-closed (safe) but sharer believes share succeeded. + +## Low + +7. Model endpoint URLs trusted by design (AGENTS.md) — no SSRF filter. Host + responsibility. +8. Dynamic SQL for embed table names validated by `EMBED_TABLE_NAME_PATTERN` / + modelKey hex — good. +9. Dynamic `dims` interpolated only after integer bounds check — good. + +## Recommended fixes + +| # | Fix | +|---|-----| +| 1 | `"memory"."edge"` in dense entity SQL + test | +| 2 | Doc: transform APIs are privileged; host grant required | +| 3 | Optional: single-transaction exclusive activate | +| 4 | Optional: return share materialization receipt to caller | + +## Attack scenarios checked (no exploit) + +- Cross-tenant document via search without grants → post-filter + tenant SQL. +- Embed table name injection → pattern reject. +- Promote another tenant’s generation → tenantId filter on transform_run; + config tenant mismatch check on promote. +- Share grant self-grant → skipped when peer === sharedBy. diff --git a/.gitignore b/.gitignore index d7f81b8..8dc025e 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ dispatch/ # Staging dirs for sibling package extracts (copy out with cp only) .staging-*/ + +# Local agent/session scratch — never commit +.corbits/ diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index d56cbac..95d91b3 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -9,7 +9,8 @@ plane and the protected routes that read and write it. The store was detachable from a larger backend, then mountable: - No memory table has a foreign key into any control-plane table — cross-refs - (`tenant_id`, `principal_id`, source refs) are plain `text`. + (`tenant_id`, `principal_id`, source refs) are plain `text`. Tables live in + the **`memory`** schema (same Postgres URL as the host is fine). - Embedding and reranking go out as plain HTTP to configured model endpoints. - Document access is Interchange grant tags on the row (`accessTags` + creator), not a private ACL engine inside this package. @@ -21,7 +22,11 @@ talks to its DocumentStore. No second server. ## Product path ``` -tools / ingestion → /api/tenants/:tenantId/memory/* → Memory plane → DocumentStore +add → ingest elements (store/chunk/embed) → process (optional, host) +``` + +``` +tools / host ingest workflow → /api/tenants/:tenantId/memory/* → Memory plane → DocumentStore ↑ Interchange auth + principal + grants ``` @@ -30,6 +35,11 @@ Mount is intentionally small. The host already has `app`, grants, and principal middleware; memory only needs to be handed those and the vector config (or an injected store). +**Ingest elements** run on the default store inside `add` (raw capture, chunks, +edges, embed). **Process** (claims, LLM link/classify) is host-owned inference, +preferably in the same workflow body as the add. Capture **feed** + distiller +helpers are optional multi-writer / backfill — not the primary path. + ## Boundaries - **Runtime**: Bun + Hono, mounted on the host app. **DB**: own pgvector diff --git a/CHANGELOG.md b/CHANGELOG.md index 8006dbe..6d944a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,14 +7,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -### Added +### Fixed + +- Feed `nextCursor` advances past the examined raw page after grant-tag + post-filter (a fully denied page no longer stalls the consumer forever). +- Distiller default system prompt uses the configured `agentId` for + `exclude_generator` / `generator_agent_id` (not only the package default id). +### Changed + +- **Product narrative:** default path is **add → ingest elements → process** + (one host pipeline). Pull feed + `createResidentDistiller` are optional + multi-writer / backfill process helpers, not the primary ingest story. + See `PRODUCT.md`, `docs/DISTILLER.md`, `docs/FEED.md`. +- **Breaking:** Postgres schema renamed from `knowledge` to **`memory`**. Fresh + installs only — drop/recreate the old schema (or rename) on existing DBs. + Citation `open.type` is now `"memory"`. +- **Breaking:** env var is `DATABASE_URL` (was `KNOWLEDGE_DATABASE_URL`); pass + `memory.databaseUrl` on config as an alternative. No deprecated alias. +- `add` (plane + HTTP) returns `{ documentId, versionId }` so share-grant audit + and provenance can name the version that carried the write. - Interchange `defineTool` factories at `@corbits/memory/tools` (`memory_add`, `memory_search`, `memory_list`) — HTTP clients for mounted hub routes with install env `memoryBaseUrl` / `memoryTenantId` / `memoryAuthToken`. Declared via `package.json` `interchange.tools` and `exports["./tools"]`. -### Changed +### Previously - **Breaking:** package and public surface renamed from `@corbits/knowledge-engine` to `@corbits/memory`. Public APIs: `createMemory` (optional `app` registers HTTP), @@ -45,15 +63,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 mini-ACL. Share sugar only mints tags. See `docs/AUTHZ-DOCUMENT-ACCESS.md`. - **Breaking:** Postgres baseline is two files (`0001_extensions` + `0002_memory_baseline`) with `access_tags` and no `visibility_*` columns. - Fresh installs only — drop/recreate the memory schema on existing DBs. + Fresh installs only — drop/recreate the `memory` schema on existing DBs. - **Breaking:** `grantStore` + `conditionRegistry` are top-level `createMemory` options (no nested `grants: { … }`). ### Added - Optional `TextExtractor` + `file` XOR `content` on `add` -- `share` sugar on `add` (maps to access tags only: owner, tenant, peers) +- `share` sugar on `add` (maps to access tags; principals also materialize + grants when the host grant store is writable); `grantsMaterialized` on the + add result when peers were requested - `access_tags` on `memory.document` (baseline schema) +- Claim-bearing schema, temporal model, transform/replay, share grants + (resident memory distillation foundation — CL-5865/5866/5872/5873) +- Living relevancy: corroboration factor from supports/contradicts edges; + strong evidence gate (CL-5867). See `docs/RELEVANCY.md` +- Capture feed: `memory.feed` + `GET .../memory/feed` with `feed_seq` cursor + (CL-5868). See `docs/FEED.md` +- **Wire attribution (CL-5870):** search hits carry additive `attribution` + (versionId, provenance, source/temporal class, createdByKind, + generatorAgentId, occurredAt/validUntil, corroboration counts, derivedFrom) +- **Retention (CL-5871):** `retention_class` on versions; plane APIs + `deprecateVersion` / `tombstoneDocument` / `hardDeleteDocument` / + `sweepEphemeral` / `setRetentionClass`; `includeDeprecated` on search. + See `docs/RETENTION.md` +- **Resident distiller (CL-5869):** `@corbits/memory/distiller` — + `createResidentDistiller({ inference })` schedule workflow + `runDistillTick` + + `buildDistilledClaim`. Claim-aware `add` (generator_agent_id, provenance, + derived_from, …). `memory_feed` tool. Feed entries include `accessTags`. + See `docs/DISTILLER.md` ### Removed diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md index 49143a3..75c7b6d 100644 --- a/IMPLEMENTATION.md +++ b/IMPLEMENTATION.md @@ -8,7 +8,7 @@ and wire shapes. For the "why standalone" / boundaries story, read ``` src/ - index.ts # createMemory / registerMemoryRoutes + index.ts # createMemory / registerMemoryRoutes + distiller re-exports mount-config.ts # MemoryConfig + loadMemoryConfig() — the mount config config.ts # EngineConfig — the core vector-plane config (db + embed + rerank) @@ -22,7 +22,14 @@ src/ mount.ts # registerMemoryRoutes (HTTP) deps.ts # RouteDeps, caller(c) (context identity), grantGuard - add.ts, search.ts, list.ts + add.ts, search.ts, list.ts, feed.ts + tools/ # Interchange defineTool factories (HTTP clients) + add.ts, search.ts, list.ts, feed.ts, client.ts, install.ts + distiller/ # Resident distiller (CL-5869) — workflow + tick helpers + index.ts # createResidentDistiller, runDistillTick, buildDistilledClaim + workflow.ts # defineWorkflow + defineAgent with memory tools + tick.ts # imperative distill tick (host injects distill()) + claim.ts # pure claim body / gate / cursor helpers db/ schema.ts # Drizzle table defs (memory.* schema) client.ts # createDb(config) -> { db (drizzle), sql (raw postgres-js) } @@ -31,6 +38,9 @@ src/ search.ts # hybridSearch and every retrieval-candidate query timeline.ts # listTimelineEvents — durable recent docs + grant-tag filter transform.ts # transform_config CRUD + runTransform (replay) + feed.ts # capture feed cursor pull (CL-5868) + retention.ts # deprecate / tombstone / sweep ephemeral (CL-5871) + share-grants.ts # peer grant materialization on share (CL-5873) core/ # framework-agnostic (chunking, embed/rerank, merge, schemas) # DocumentStore adapters / tools live as sibling packages (not in this tree): # @corbits/mem0-memory-adapter → github.com/corbitsdev/corbits-mem0-memory-adapter @@ -196,19 +206,66 @@ Lightweight graph rows. `memory_entity` has no unique constraint; dedupe on `(tenant_id, kind, identifiers)` is done in application code (`upsertEntity` in `capture.ts`, an exact-match linear scan per kind). Same for `memory_edge` (dedupe on the full `(tenant_id, rel, from, to)` tuple, -`upsertEdge`). `rel` is constrained (DB CHECK + arktype) to -`'about'|'produced_by'|'links'|'parent'|'mentions'|'waiting_on'`; `from_type`/ -`to_type` to `'document'|'entity'|'native'`. +`upsertEdge`). `rel` is constrained (DB CHECK + arktype, single source of +truth in `src/core/enums.ts`) to +`mentions|about|authored_by|involves|part_of|derived_from|supports|contradicts|supersedes`; +`from_type`/`to_type` stored in the DB are +`document|version|chunk|entity`. Adapter-facing edge hints may also use +`native` as a planning-time principal ref; capture resolves it to an +`entity` row (`kind=principal`) before insert. A lockstep test asserts the +migration CHECK sets match the TS constants. + +### Provenance and lineage (claim-bearing substrate) + +Two axes on `memory.version`, orthogonal to ranking priors: + +| Column / field | Values | Meaning | +| --- | --- | --- | +| `provenance` | `stated` \| `inferred` \| `unknown` | How content was obtained. Capture defaults to `stated`; distilled claims write `inferred`. Existing rows default `unknown`. | +| `source_class` (lineage) | `native` \| `imported` \| `derived` | Data lineage. Adapters write `native` (or `imported` for bulk import); distilled claims write `derived` via `AdaptedDocument.lineageClass`. | + +Ranking priors (`AdaptedDocument.sourceClass`: `native|thread|channel|call|record`) feed `computeAuthority` only and are **not** written to the version `source_class` column — that was a latent CHECK violation fixed when the axes were split. + +A derived claim is a normal version with `provenance: inferred`, `lineageClass: derived`, and a `derived_from` edge to the source version (or document). Core never runs inference; it only accepts the shape. + +### Temporal model + +See `docs/TEMPORAL.md`. On `memory.version`: + +| Column | Role | +| --- | --- | +| `occurred_at` | Effective time the content refers to | +| `ingested_at` | When the plane learned it (no separate `asserted_at`) | +| `temporal_class` | `event` \| `deadline` \| `state` \| `lesson` — ranking prior | +| `valid_from` / `valid_until` | Optional validity window | + +Search multiplies fused scores by `temporalRecencyMultiplier` (class-aware). Timeline and search both filter `generation` (default live) so replay rows never leak into live views. ### `memory_embed_model` -Per-tenant registry of which embed model is currently active, and the -dimensionality it was discovered at (`discoverModelDims`/`probeEmbedDims` — -dims are **never** hard-coded, always probed live against the endpoint). -Unique on `(tenant_id, model_key)`, where `model_key` is -`sha256(baseUrl|modelId).slice(0,16)` (`computeModelKey`). "Active" means the -most-recently-`updated_at` row with `status = 'active'` for that tenant -(`resolveActiveEmbedTable`) — there is no per-generation embed-model scoping -(see "Known limitation" under Raw + replay below). +Per-tenant registry of embed models and their discovered dimensionality +(`discoverModelDims`/`probeEmbedDims` — dims are **never** hard-coded, always +probed live against the endpoint). Unique on `(tenant_id, model_key)`, where +`model_key` is `sha256(baseUrl|modelId).slice(0,16)` (`computeModelKey`). + +Two write paths (CL-5872): + +- **`ensureEmbedModel`** — upserts the registry row with `status = 'ready'`, + creates the per-model table + indexes. **Never** flips the tenant's active + dense table. Used by staged `runTransform` when the config's embed model + differs from live. +- **`activateEmbedModel`** — `ensure` then `UPDATE … SET status = 'active'`. + Used only by the **live capture** path and by **`promoteGeneration`** when + cutover should make the generation's embed model serve live dense search. + +"Active" for live dense search is the most-recently-`updated_at` row with +`status = 'active'` (`resolveActiveEmbedTable`). Replay dense search uses +`resolveEmbedTableByModelKey` against the owning transform config's embed +`model_key` — ready or active — so staged generations never steal live. + +Optional `archived_live_generation` and `archived_live_model_key` on +`transform_run` record which generation and dense model held live rows before +promote (for demote/rollback of both corpus and dense search). + ### Dynamic per-model vector tables: `memory_embedding_` Not in `db/schema.ts` (no fixed shape — dimensionality varies by model) and @@ -248,7 +305,7 @@ at activation instead. The dense query's `ORDER BY` is generated by `cosineDistanceExpr` from the same module so it always matches the indexed expression. The table name is validated against `EMBED_TABLE_NAME_PATTERN` -(`/^memory_embedding_[a-f0-9]{16}$/`) both when computed and again every +(`/^"memory"\."embedding_[a-f0-9]{16}"$/`) both when computed and again every time it's read back from `memory_embed_model`, before ever being string-interpolated into raw SQL — this is the only place in the codebase a computed identifier is spliced into DDL/DML. @@ -356,12 +413,38 @@ Entry point, one query in, one ranked/citable hit list out. `k` is clamped to is only accepted if `kinds` or `entityIds` is provided (structured-filter-only search); otherwise it throws `MemorySearchInputError` (400). +**Living relevancy (CL-5867):** after fusion, `attachCorroborationCounts` loads +`supports`/`contradicts` edge counts per version. Ranking multiplies by +`corroborationFactor` (bounded [0.7, 1.3]); evidence:strong also requires the +gate in `core/corroboration.ts` (stated human **or** support count ≥ floor). +Capture-time `authority` is never rewritten. See `docs/RELEVANCY.md`. + +**Wire attribution (CL-5870):** `attachDerivedFrom` loads `derived_from` edges +onto candidates; `toHit` emits provenance, source/temporal class, occurred_at, +valid_until, corroboration counts, and derived_from. The plane maps these into +additive `SearchItem.attribution` (and `DocumentStoreSearchItem.attribution`). +**List/timeline** stays document-title oriented and does not attach the full +attribution block (version-level fields are search/feed concerns). + +**Retention (CL-5871):** `memory.version.retention_class` + migration +`0007_retention.sql`. Plane helpers in `services/retention.ts` +(`deprecateVersion`, `tombstoneDocument`, `hardDeleteDocument`, +`sweepEphemeral`, `setRetentionClass`). Search accepts `includeDeprecated` +so lexical/dense can include `status IN ('active','deprecated')`. See +`docs/RETENTION.md`. + +**Capture feed (CL-5868):** `memory.feed({ after, limit, excludeGenerator? })` +and `GET .../memory/feed` pull live versions ordered by `feed_seq` (migration +`0006_capture_feed.sql`). Grant-checked like search. See `docs/FEED.md`. + 1. **Generation resolution** — `generation` defaults to `LIVE_GENERATION`. For any non-live generation, `resolveGenerationSearchParams` (transform.ts) - looks up the owning `transform_run` → `transform_config` and pulls its - tuning knobs (`authorityWeight`, `recencyHalfLifeDays`, `mmrLambda`, - `overfetch`, `rerank`); every field it doesn't supply falls back to the - engine's own defaults. Live search never pays for this lookup. + looks up the owning `transform_run` → `transform_config` **for the search + tenant** and pulls its tuning knobs (`authorityWeight`, + `recencyHalfLifeDays`, `mmrLambda`, `overfetch`, `rerank`); every field it + doesn't supply falls back to the engine's own defaults. Live search never + pays for this lookup. Cross-tenant generation ids resolve to `null` + (engine defaults) so embed overrides (including `apiKey`) cannot leak. 2. **Lexical channel** — `fetchLexicalCandidates`: Postgres full-text search (`ts_rank` against `plainto_tsquery` in the configured `FTS_LANGUAGE`, bound as a `regconfig` parameter, over @@ -372,9 +455,11 @@ search); otherwise it throws `MemorySearchInputError` (400). `kinds` and/or `entityIds` (via a sub-select against `memory_edge`). Overfetches up to `overfetchLimit` rows, non-deduped, per-chunk. 3. **Dense channel** — `fetchDenseCandidates`: embeds the query - (`embedTexts`), resolves the tenant's single active embedding table - (`resolveActiveEmbedTable` — **not** generation-scoped, see limitation - below), runs a raw-SQL cosine-distance ANN query via `cosineDistanceExpr` + (`embedTexts`), resolves the dense table: + - live generation → `resolveActiveEmbedTable` (tenant active model) + - staged generation → `resolveEmbedTableByModelKey` for the transform + config's embed model (ready or active; never activates) + runs a raw-SQL cosine-distance ANN query via `cosineDistanceExpr` (`e.embedding <=> $vector` up to 2000 dims, or the matching `(e.embedding::halfvec(N)) <=> $vector::halfvec(N)` expression above that so the halfvec HNSW index is used) @@ -420,7 +505,7 @@ search); otherwise it throws `MemorySearchInputError` (400). entity edges; `toHit` builds the wire `SearchHit` (citation/open-target resolution via `openTarget`, mapping known adapters — `artifact`, `task`, `workflow_run`, `mail` — to a deep-linkable `{type, id}`, else a generic - `{type: "knowledge", id: documentId}`). + `{type: "memory", id: documentId}`). 11. **Evidence** — `deriveHybridEvidence`: `"none"` if zero hits; `"weak"` if the lexical channel contributed zero rows (a dense-only result never reports `"strong"`); otherwise `deriveEvidence` on the lexical rows — @@ -450,28 +535,41 @@ writes to. A `transform_config` is a named/versioned recipe own trust boundary, never a blind `JSON.parse`), then calls `deriveFromRawCapture` with the config's own chunker (`chunkTokenRecursive` with the config's caps) and embed client config, - targeting the run's `generation` — never the live one. + targeting the run's `generation` — never the live one. Embed tables are + **ensured** (`ensureEmbedModel`), never activated, so live dense search is + untouched until an explicit promote. 5. On completion, updates the run row: `status: 'completed'`, `rawCount`, `versionCount`. On any exception mid-loop, catches it, logs it, and marks the run `'failed'` with `error` set — `runTransform` itself never throws to its caller; callers always get a run summary. 6. `resolveGenerationSearchParams` is how `hybridSearch` later maps a generation back to its config's search-tuning knobs (authority weight, - recency half-life, MMR λ, overfetch, rerank config). - -**Documented limitation — per-generation embed-model isolation does not -exist.** `resolveActiveEmbedTable` picks the tenant's single -most-recently-`updated_at` active model, with no `generation` argument at -all. If a replay's `transform_config.embed` points at a *different* -`(baseUrl, modelId)` than the live capture path currently uses, running that -replay makes its model the tenant's active dense-channel table for **every** -generation's search, including live's, from that point forward. The `search.ts` -comment on `fetchDenseCandidatesArgs.generation` states this explicitly; -`transform.ts`'s replay test in `e2e.integration.test.ts` sidesteps it by -reusing the exact same embed endpoint/model as the live capture. Scoping -activation per-generation is out of scope for the current replay-pipeline -implementation — callers replaying under a different embed model should do -so knowing it will flip the tenant's live dense channel too. + recency half-life, MMR λ, overfetch, rerank config) **and** dense + `modelKey` for generation-scoped table resolution. + +**Promote / demote (staged cutover):** + +- `promoteGeneration` — requires `status = 'completed'`. Snapshots the + pre-promote active `model_key`, activates the staged embed model (sole + active for the tenant; peers demoted to `ready`), then swaps generation + tags (`live` → archive, staged → `live`). Records + `archived_live_generation` + `archived_live_model_key` for demote. If the + version swap fails after activate, re-activates the prior model_key (or + clears active when there was no prior). +- `demoteGeneration` — re-activates `archived_live_model_key` (fail-closed if + the registry row is gone), or clears all active models when no prior was + recorded, then restores archive → live and staged corpus back onto the run + generation. + +Plane methods (engine DocumentStore only): `createTransformConfig`, +`listTransformConfigs`, `runTransform`, `promoteGeneration`, +`demoteGeneration` on `Memory`. Custom/fake stores omit these methods. + +**Host privilege:** transform/promote/demote take `tenantId` (and config/run +ids) only — no principal, no HTTP routes. They rewrite live generation and the +active dense embed model for a tenant. The host must treat them as admin +operations (grant-gate before calling); never expose them unauthenticated to +end-user agents. ## Mounted routes @@ -486,7 +584,7 @@ Each route is guarded with `grantGuard(deps, action)`, which applies the host's | Method + path | Grant action | Request body | Response | |---|---|---|---| -| `POST /api/tenants/:tenantId/memory/add` | `add` | `{ title, text, access_tags?, share? }` | `200 { documentId }`; `400` on validation | +| `POST /api/tenants/:tenantId/memory/add` | `add` | `{ title, text, access_tags?, share? }` | `200 { documentId, versionId }`; `400` on validation | | `POST /api/tenants/:tenantId/memory/search` | `search` | `{ query, limit?, kinds?, entity_ids?, sources?, includeEvidence? }` (limit 1–50; `kinds`/`entity_ids`/`sources` narrow retrieval before fusion; unset or `[]` = unfiltered; `includeEvidence` adds a short evidence string when true) | `200 { items[], evidence?, degraded? }`; `400` on bad input | | `GET /api/tenants/:tenantId/memory/list` | `search` | query `?limit=` (1–100, string on the wire) | `200 { events: [{ at, title, source, tenantId, principalId }] }` — durable recent documents for the caller's scope, filtered with grant-tag access (`canAccessDocument`). One event per document (active live version). | @@ -498,8 +596,26 @@ mounted routes with install env (`memoryBaseUrl`, `memoryTenantId`, needs `memory:add` and/or `memory:search` grants; Bearer token only (no session cookie path); tool results are JSON strings; pass `AbortSignal` if you need hang protection — the client has no default timeout. OpenAPI→MCP remains an optional -host bridge. The plane surface is only `add` / `search` / `list` (plus `close`); -inference stays on the host. +host bridge. The plane surface is `add` / `search` / `list` / `close`, plus +optional transform methods when backed by the engine DocumentStore +(`createTransformConfig`, `listTransformConfigs`, `runTransform`, +`promoteGeneration`, `demoteGeneration`). Inference stays on the host. + +### Share materialization (CL-5873) + +`share.principals` on `add` still mints owner tags, and when the host grant +store implements `WritableGrantStore.putGrant`: + +1. Appends `memory.doc:` to the document's `access_tags`. +2. Writes one allow/`search` grant per peer on that resource, origin + `system`, with `conditions.memoryShare` audit payload. +3. `resolveGrantConfig` merges `MEMORY_SHARE_CONDITION_REGISTRY` so those + condition keys are not fail-closed-skipped by `@intx/authz`. + +Without a writable store: tags only + warn log (peers need host grants). +Audience widening uses `splitAudienceWiden` (write-narrow-then-widen) and +`shareWidenReceipt` on version attributes after source-owner approval. +Ask-on-read remains design-only (fail-closed). diff --git a/PRODUCT.md b/PRODUCT.md index 94dee05..105f3cc 100644 --- a/PRODUCT.md +++ b/PRODUCT.md @@ -4,8 +4,27 @@ Memory for Interchange hubs: durable documents, hybrid search, recent list. **You mount it on the hub (~5 lines). That exposes protected routes. Agents and ingestion modules call those routes.** Workbench and coding agents are -clients — not owners of ingestion or auth. Inference is host-owned (call your -model, then `add` / `search`); core does not ship an answer endpoint. +clients — not owners of auth. Inference stays host-injected. + +## Default pipeline (locked) + +```text +add → ingest elements → process (optional) +``` + +| Stage | Meaning | Where | +| --- | --- | --- | +| **add** | Something arrives (agent tool, host job, webhook body) | Caller → `memory.add` / `POST …/memory/add` | +| **ingest elements** | Normalize → raw capture → chunks / edges → embed → search-ready | Default `DocumentStore` capture path (sync on `add`) | +| **process** | Optional brain work: classify, claims, links, forget | Host workflow / injected inference — same run as ingest when possible | + +Preferred host shape: **one ingest workflow** receives the event, calls `add` +(ingest elements), then runs process steps in the same body (or a child step). +No pull feed required on that path — the workflow already has the payload. + +**Pull feed + resident distiller** are optional: multi-writer backfill, replay, +or polish when other code also `add`s outside the ingest workflow. See +`docs/DISTILLER.md` and `docs/FEED.md`. ## Shape (locked) @@ -18,15 +37,23 @@ never creates one; it mounts onto yours. | `loadMemoryConfig()` | Config from env | | `runMemoryMigrations(url)` | Apply pgvector schema | | `registerMemoryRoutes` | Low-level HTTP only (optional) | -| `@corbits/memory/tools` | Interchange `defineTool` factories (`memory_add` / `memory_search` / `memory_list`) | +| `@corbits/memory/tools` | Interchange tools (`memory_add` / `search` / `list` / `feed`) | +| `@corbits/memory/distiller` | Optional process helpers: `runDistillTick`, `createResidentDistiller` | ### Verbs | Method | HTTP | Grant | Meaning | | --- | --- | --- | --- | -| `add` | `POST /api/tenants/:tenantId/memory/add` | `memory:add` | Capture a document | -| `search` | `POST /api/tenants/:tenantId/memory/search` | `memory:search` | Hybrid retrieval (+ optional live sources) | +| `add` | `POST /api/tenants/:tenantId/memory/add` | `memory:add` | Ingest: capture + derive (chunk/embed on default store) | +| `search` | `POST /api/tenants/:tenantId/memory/search` | `memory:search` | Hybrid retrieval (+ optional live sources); hits may include additive `attribution` | | `list` | `GET /api/tenants/:tenantId/memory/list` | `memory:search` | Recent documents for the principal | +| `feed` | `GET /api/tenants/:tenantId/memory/feed` | `memory:search` | Cursor pull of new live versions (optional multi-writer / backfill) | + +Engine-only plane helpers (no HTTP yet): transform/replay, retention +(`deprecateVersion` / `tombstoneDocument` / … — see `docs/RETENTION.md`), +share-grant materialization. Process helpers: +`createResidentDistiller` / `runDistillTick` (`docs/DISTILLER.md`) — host +injects inference; not the default ingest path. Identity is always **`principalId` + `tenantId`** on the plane. HTTP routes never take body identity — they read `c.get("principal")` from Interchange @@ -35,7 +62,7 @@ context. ### How it is used ``` -Agent / ingestion module +Agent / host ingest workflow │ tool call or host worker │ → POST|GET /api/tenants/:tenantId/memory/* │ authenticated by Interchange (session | API key | MCP OAuth) @@ -58,8 +85,10 @@ Agent / ingestion module workflow with env credentials (`memoryBaseUrl`, `memoryTenantId`, `memoryAuthToken`). Tools HTTP-call the mounted routes; identity is the hub-authenticated principal. OpenAPI→MCP remains an optional host bridge. -3. **Ingestion** — host modules (webhooks, batch jobs) call the routes or the - returned plane with a resolved principal. +3. **Ingestion** — preferred: one host workflow (or module) does + **add → ingest elements → process**. Mechanical ingest is inside `add` on + the default store; process (claims / links) is host-injected inference in + the same pipeline when you want a company brain. ### Ports @@ -76,6 +105,7 @@ stores, Linear tools. Core never imports vendor SDKs. - No auth, API keys, OAuth, webhooks, SPA, or standalone server in core. - No answer/generation endpoint — host owns inference. - Workbench is a client, not required. +- Core does not run the ingest workflow process — the host does. **Default durable store:** Postgres via `DATABASE_URL`, tables under the **`memory`** schema. When diff --git a/bun.lock b/bun.lock index eeeb56d..a2bdc12 100644 --- a/bun.lock +++ b/bun.lock @@ -9,6 +9,7 @@ "@intx/authz": "0.2.2", "@intx/hub-api": "0.2.2", "@intx/log": "0.2.2", + "@intx/workflow": "0.2.2", "arktype": "^2.1.29", "drizzle-orm": "^0.45.1", "hono": "^4.9.0", diff --git a/docs/AUTHZ-DOCUMENT-ACCESS.md b/docs/AUTHZ-DOCUMENT-ACCESS.md index 6f1ee3e..fad88c9 100644 --- a/docs/AUTHZ-DOCUMENT-ACCESS.md +++ b/docs/AUTHZ-DOCUMENT-ACCESS.md @@ -78,18 +78,28 @@ There is **no** `share.private` key. Owner-only is the default when `share` is o Tag minting is **not** grant minting. For peer share to work in product: 1. When Alice adds with `share: { principals: ["bob"] }`, the document is tagged - `memory.owner:alice` and `memory.owner:bob`. -2. Bob sees it only if the host has granted Bob `search` on `memory.owner:bob` - (or a pattern that matches). **Recommended host bootstrap:** every principal - receives `search` (and optionally `add` side-effects as you prefer) on - `memory.owner:` at signup, or a single pattern grant such as - `memory.owner:*` only if that matches your tenancy model. -3. Space/tenant tags work the same way: host must issue grants on - `memory.space:eng` / `memory.tenant:` for non-creators to match. - -Without (2), `share.principals` is a silent no-op for peers (fail-closed; looks -like empty search). Document this in host mount guides — do not reintroduce a -document mini-ACL in this package. + `memory.owner:alice` and (after insert) `memory.doc:`. +2. When the host grant store implements `WritableGrantStore.putGrant`, memory + **materializes** an allow/`search` grant for Bob on `memory.doc:` + (origin `system`, conditions carry `memoryShare` / `sharedBy` / `sourceVersionId` + for audit). Peers can then pass `canAccessDocument` without a separate + bootstrap grant on `memory.owner:bob`. +3. Without a writable grant store, tags alone are written and peers still need + host-side grants (fail-closed empty search). Log warns on this path. +4. `share.tenant` / `share.tags` still only mint tags — hosts issue role/pattern + grants on those resources (no auto-principal grants). + +### Audience widening (write-narrow-then-widen) + +Distiller / claim writes that propose tags **beyond** the source document's +audience must not silently widen: + +1. `splitAudienceWiden(sourceTags, proposed)` → write with `allowed` only. +2. After **source-owner** approval, append `needsApproval` tags and materialize + any peer grants; store `shareWidenReceipt` on version attributes. + +Ask-on-read (`authorize` effect `"ask"`) remains **fail-closed** in +`canAccessDocument` (design-only; no flag in v1). **Removed:** `visibility: { mode: private|principals|tenant }`, `blockPrincipalIds`, product `acl.mode` / `acl.allow` / `acl.block` as security. diff --git a/docs/DISTILLER.md b/docs/DISTILLER.md new file mode 100644 index 0000000..66f4138 --- /dev/null +++ b/docs/DISTILLER.md @@ -0,0 +1,118 @@ +# Process helpers (distiller) + +Default product path is **add → ingest elements → process** in one host +pipeline (`PRODUCT.md`). Mechanical ingest (raw → chunk → embed) already runs +inside `memory.add` on the default store. + +This package’s distiller exports are **optional process helpers** for when +you need LLM claim extraction **outside** that single pipeline: + +- multi-writer: other agents also `add` and you want a backfill worker +- replay / catch-up over a cursor +- fail-soft polish decoupled from the write that ingested the raw note + +They are **not** the primary “how memory is ingested” story. + +## Preferred: process in the same ingest workflow + +```text +onTrigger / host job + 1. receive source event (or fetch) + 2. memory_add // ingest elements + 3. process (optional) // claims / links — same body or child step +``` + +No feed cursor required: the workflow already has the payload. Host injects +inference; tools only need `memory_add` / `memory_search` (and grants). + +Helpers still useful in-process: + +| Export | Use | +| --- | --- | +| `buildDistilledClaim` | Wire body with `generator_agent_id`, `provenance=inferred`, `derived_from` | +| `RESIDENT_DISTILLER_AGENT_ID` | Stable generator id if you write claims | + +## Optional: multi-writer / backfill (`runDistillTick`) + +When other writers also `add`, drain new versions with the capture feed: + +```ts +import { runDistillTick } from "@corbits/memory/distiller"; +import { createMemoryHttpClient } from "@corbits/memory/tools"; + +const client = createMemoryHttpClient({ + baseUrl: process.env.MEMORY_BASE_URL!, + tenantId: process.env.MEMORY_TENANT_ID!, + authToken: process.env.MEMORY_AUTH_TOKEN!, +}); + +let cursor = 0; +const result = await runDistillTick({ + client, + after: cursor, + distill: async (entry) => { + // call your model — return skip | poison | write + return { + action: "write", + title: "Claim", + text: "…", + temporalClass: "lesson", + }; + }, +}); +cursor = result.nextCursor; // persist +``` + +Inference is **always injected** (`distill` callback or host agent sources). +The package never embeds a model. + +## Optional: schedule workflow scaffold (`createResidentDistiller`) + +Scaffold for hosts that still want a deployed agent with memory tools + +system prompt (loop-safety, access-tag copy). Prefer wiring **process next to +add** in your ingest workflow; use this for backfill-style residency only. + +```ts +import { createResidentDistiller } from "@corbits/memory/distiller"; + +const { workflow, generatorAgentId } = createResidentDistiller({ + inference: { + sources: [{ provider: "openai", model: "gpt-4.1-mini" }], + }, +}); +// Deploy only if you need a multi-writer pull consumer — not default ingest. +``` + +## Substrate (plane) + +| Piece | Where | +| --- | --- | +| Ingest on add | capture path — raw + chunks + embed | +| Capture feed (cursor) | `memory.feed` — [FEED.md](./FEED.md) (backfill / multi-writer) | +| Claim identity on add | `generator_agent_id`, `provenance`, `lineage_class`, `derived_from` | +| Wire attribution on search | `SearchItem.attribution` | +| Retention / forgetting | [RETENTION.md](./RETENTION.md) | +| Tools | `@corbits/memory/tools` | + +## Grant manifest (process principal) + +Installer discovery (not live grants): + +- `package.json` → `interchange.grantRequirements` +- typed SSOT: `MEMORY_GRANT_REQUIREMENTS` / `MEMORY_CAPABILITY_IDS` from + `@corbits/memory` (or `@corbits/memory/tools`) + +Minimum capabilities: + +- `memory:add` (claim or note writes) +- `memory:search` (corroboration + feed if using backfill) + +Deploy materializes these onto the workflow principal. Copy `accessTags` from +the source onto claim writes — never mint broader tags. + +## Out of scope + +- Host deploy pipeline / secrets +- Push outbox (optional later; not required if process is in-pipeline) +- Core-owned ingest workflow process (host owns that) +- Automatic supports/contradicts edge minting beyond `derived_from` on add diff --git a/docs/FEED.md b/docs/FEED.md new file mode 100644 index 0000000..e882c41 --- /dev/null +++ b/docs/FEED.md @@ -0,0 +1,52 @@ +# Capture feed + +Stateless, cursorable pull of new **versions** for **optional** multi-writer +backfill / process workers (CL-5868). + +**Default product path does not need this.** Prefer +**add → ingest elements → process** in one host pipeline (`PRODUCT.md`): the +workflow already has the payload, so no pull cursor. + +Use the feed when: + +- other agents or modules also `add` outside your ingest workflow +- you need catch-up / replay over a durable ordering key +- process is intentionally decoupled from the write (fail-soft polish) + +## Phase 1 — pull (implemented) + +``` +memory.feed({ tenantId, principalId, after?, limit?, excludeGenerator? }) +``` + +| Field | Meaning | +|-------|---------| +| `after` | Last consumed `feedSeq` (exclusive). Omit/0 = from start. | +| `limit` | Page size (bounded). | +| `excludeGenerator` | Skip versions with this `generator_agent_id` (loop-safe). | + +- Ordered by `feed_seq` ascending (Postgres `bigserial` on `memory.version`). +- **Live generation only** — same rule as default search. +- Capability: `memory` / `search` (same as list/retrieve). +- Document access: grant-tag post-filter identical to search. +- **Cursor advances past the examined raw page**, even when the access filter + returns zero entries. Using the last *allowed* `feedSeq` would stall a + consumer on a fully denied page forever. + +Cursor storage is the **consumer's** job (workflow run state). + +HTTP: `GET /api/tenants/:tenantId/memory/feed?after=&limit=&exclude_generator=` + +## Phase 2 — push (design only) + +Post-commit outbox row keyed by `feed_seq` + host dispatcher that mails the +deployment address with version ids. **Not implemented** in core. Phase 1 +`feed_seq` is the ordering key so Phase 2 is additive. Only relevant if process +stays out-of-band from the writer. + +## Non-goals + +- In-core cron or push dispatcher +- Bypassing grant tags for “tenant brain” +- Including replay generations in the default feed +- Replacing the default **add → ingest → process** host pipeline diff --git a/docs/RELEVANCY.md b/docs/RELEVANCY.md new file mode 100644 index 0000000..f230778 --- /dev/null +++ b/docs/RELEVANCY.md @@ -0,0 +1,44 @@ +# Claim relevancy (corroboration) + +Living relevancy for claim-bearing versions. Capture-time **`authority`** remains +a frozen snapshot (`computeAuthority` at write). Search derives a separate +**corroboration factor** from graph edges and multiplies ranking with it. + +## Edges + +| `rel` | Effect | +|-------|--------| +| `supports` | Independent source backs the target claim version — factor up | +| `contradicts` | Disagreement signal — factor down; **no** auto-delete/supersede | + +Counts are edges with `to_type = 'version'` and `to_ref = `. +Writers (typically the resident distiller) attach `supports` / `contradicts` +hints on capture; core does **not** decide claim sameness. + +## Ranking + +`corroborationFactor({ supports, contradicts })` ∈ **[0.7, 1.3]** (same envelope +as authority/recency boosts). Neutral `1.0` when both counts are zero. + +Effective authority for rank priors: + +``` +effectiveAuthority = clamp01(captureAuthority × corroborationFactor) +``` + +## Evidence: strong + +After relevance floors clear, **strong** also requires: + +1. Capture authority ≥ `AUTHORITY_STRONG_FLOOR` (0.3), and +2. Either: + - `provenance: stated` + `created_by_kind: human`, or + - support count ≥ `CORROBORATION_STRONG_FLOOR` (default **2**) + +Constants live in `src/core/corroboration.ts`. + +## Non-goals + +- Embedding-similarity merge of claims +- Silent resolution of human disagreement +- Inference inside `@corbits/memory` diff --git a/docs/RETENTION.md b/docs/RETENTION.md new file mode 100644 index 0000000..fe230cd --- /dev/null +++ b/docs/RETENTION.md @@ -0,0 +1,32 @@ +# Retention classes (CL-5871) + +Versions carry a **retention class** orthogonal to temporal ranking class +(`temporal_class`) and lineage (`source_class` / `provenance`). + +| Class | Intent | +| --- | --- | +| `durable` | Long-lived claims; hard-delete blocked until tombstoned | +| `standard` | Default working memory | +| `ephemeral` | Short TTL; sweeper hard-deletes past `valid_until` (or 7d from `ingested_at`) | +| `source_only` | Keep raw capture; derived versions may be dropped by host policy | + +Schema: `memory.version.retention_class` (migration `0007_retention.sql`). +CHECK constraint `version_retention_class_check` stays lockstep with +`RETENTION_CLASSES` in `src/core/enums.ts`. + +## Write paths + +| Verb | Plane API | Effect | +| --- | --- | --- | +| Deprecate | `memory.deprecateVersion` | `status=deprecated`, `deprecated_at` / reason | +| Tombstone | `memory.tombstoneDocument` | All active/deprecated/superseded versions → `tombstoned`; chunk text redacted to `[redacted]` | +| Hard delete | `memory.hardDeleteDocument` | Deletes document row (cascade); **refuses** if any non-tombstoned version is `durable` | +| Sweep | `memory.sweepEphemeral` | Auto-deprecates ephemeral versions past `valid_until` (or 7d from `ingested_at`); host schedules, core is cron-free | +| Set class | `memory.setRetentionClass` | Update `retention_class` on a version | + +Search and feed exclude non-active (and non-superseded for feed) rows by +default. Pass `includeDeprecated: true` on search to retrieve deprecated +versions intentionally (ops / audit). Hard-delete is a separate explicit +verb — TTL never hard-deletes. + +Service module: `src/services/retention.ts`. diff --git a/docs/TEMPORAL.md b/docs/TEMPORAL.md new file mode 100644 index 0000000..d024208 --- /dev/null +++ b/docs/TEMPORAL.md @@ -0,0 +1,57 @@ +# Temporal model + +How `memory.version` times and ranking classes work. Implementation lives +in capture (write), hybrid-search (rank), and search/timeline (query filters). + +## Four times (two stored, two derived) + +| Concept | Column / source | Meaning | +| --- | --- | --- | +| **Effective time** | `occurred_at` (required) | When the content *refers to*: event moment, state effective time, or when a deadline was established. Not dual-meaning — one meaning applied across classes. | +| **Ingestion / assertion** | `ingested_at` | When the memory plane learned the content (capture or distill write). There is no separate `asserted_at`. | +| **Validity start** | `valid_from` (nullable) | Optional window start for state/deadline claims. | +| **Validity end** | `valid_until` (nullable) | Optional window end. Required in practice for useful `deadline` ranking. | + +## `temporal_class` + +SSOT: `TEMPORAL_CLASSES` in `src/core/enums.ts`. Stored on **version** (not +document) so successive versions can change class. + +| Class | Recency prior | Notes | +| --- | --- | --- | +| `event` | Exponential decay from `occurred_at` (30-day half-life default) | Default for raw captures; preserves pre-model ranking. | +| `deadline` | Neutral far out; urgency ramp in a 7-day lookahead before `valid_until`; floor (0.7) after expiry | Still history-retrievable after expiry — not deleted. | +| `state` | Constant 1.0 while `status='active'` | Default for `provenance='inferred'` (distilled claims). Supersede via capture status, not recency. | +| `lesson` | Constant 1.0 | Always explicit; never a default. | + +Defaults at write: + +- Explicit `AdaptedDocument.temporalClass` wins. +- Else if `provenance === 'inferred'` → `state`. +- Else → `event`. + +## Ranking integration + +`applyBoosts` in `src/services/search.ts` multiplies the fused score by +`authorityBoostMultiplier × temporalRecencyMultiplier`. Lexical and dense +candidate queries both select `temporal_class` and `valid_until`. + +Deadline formula (module constant `DEADLINE_LOOKAHEAD_MS = 7d`): + +- `valid_until` null → 1.0 +- remaining ≤ 0 → `BOOST_MULTIPLIER_MIN` (0.7) +- remaining ≥ lookahead → 1.0 +- else ramp 1.0 → ~1.3 as the deadline approaches + +## Timeline generation filter + +Timeline joins only **active** versions in the requested **generation** +(default `live`). Replay-generation rows must not appear in the default +timeline (same rule as hybrid search). + +## Supersedes + +Unchanged: capture sets `status='superseded'` and `supersedes_version_id` +together. Temporal ranking does not write status. `state` “no decay until +superseded” means constant recency while active; superseded rows drop out of +live search/timeline via the status filter. diff --git a/migrations/0003_claim_bearing.sql b/migrations/0003_claim_bearing.sql new file mode 100644 index 0000000..94e38a4 --- /dev/null +++ b/migrations/0003_claim_bearing.sql @@ -0,0 +1,13 @@ +-- Claim-bearing substrate: provenance mode on version. +-- Edge rel / ref-type CHECKs already match the canonical set in 0002; +-- this migration only adds how-content-was-obtained (stated vs inferred). + +ALTER TABLE "memory"."version" + ADD COLUMN IF NOT EXISTS "provenance" text NOT NULL DEFAULT 'unknown'; + +ALTER TABLE "memory"."version" + DROP CONSTRAINT IF EXISTS "version_provenance_check"; + +ALTER TABLE "memory"."version" + ADD CONSTRAINT "version_provenance_check" + CHECK ("provenance" IN ('stated', 'inferred', 'unknown')); diff --git a/migrations/0004_temporal_model.sql b/migrations/0004_temporal_model.sql new file mode 100644 index 0000000..2e7170d --- /dev/null +++ b/migrations/0004_temporal_model.sql @@ -0,0 +1,25 @@ +-- Temporal model: ranking class + validity window on version. +-- See docs/TEMPORAL.md. No asserted_at — occurred_at is effective time; +-- ingested_at is when the memory plane learned the content. + +ALTER TABLE "memory"."version" + ADD COLUMN IF NOT EXISTS "temporal_class" text NOT NULL DEFAULT 'event'; + +ALTER TABLE "memory"."version" + DROP CONSTRAINT IF EXISTS "version_temporal_class_check"; + +ALTER TABLE "memory"."version" + ADD CONSTRAINT "version_temporal_class_check" + CHECK ("temporal_class" IN ('event', 'deadline', 'state', 'lesson')); + +ALTER TABLE "memory"."version" + ADD COLUMN IF NOT EXISTS "valid_from" timestamp; + +ALTER TABLE "memory"."version" + ADD COLUMN IF NOT EXISTS "valid_until" timestamp; + +-- Distilled claims (inferred provenance) default to state ranking. +UPDATE "memory"."version" + SET "temporal_class" = 'state' + WHERE "provenance" = 'inferred' + AND "temporal_class" = 'event'; diff --git a/migrations/0005_transform_promote.sql b/migrations/0005_transform_promote.sql new file mode 100644 index 0000000..2631932 --- /dev/null +++ b/migrations/0005_transform_promote.sql @@ -0,0 +1,10 @@ +-- Promote / demote bookkeeping for staged transform generations (CL-5872). +-- archived_live_generation holds the generation tag assigned to the prior +-- live corpus during promote, so demote can swap back without data loss. +-- archived_live_model_key holds the pre-promote active embed model_key so +-- demote can restore dense search to the table that holds restored vectors. + +ALTER TABLE "memory"."transform_run" + ADD COLUMN IF NOT EXISTS "archived_live_generation" text, + ADD COLUMN IF NOT EXISTS "archived_live_model_key" text, + ADD COLUMN IF NOT EXISTS "promoted_at" timestamp with time zone; diff --git a/migrations/0006_capture_feed.sql b/migrations/0006_capture_feed.sql new file mode 100644 index 0000000..82c59d4 --- /dev/null +++ b/migrations/0006_capture_feed.sql @@ -0,0 +1,9 @@ +-- Capture feed: monotonic commit marker for exactly-once pull (CL-5868). +-- feed_seq doubles as the Phase-2 outbox ordering key (design only; no push). + +ALTER TABLE "memory"."version" + ADD COLUMN IF NOT EXISTS "feed_seq" bigserial; + +-- Live-generation drain path: tenant + generation + cursor. +CREATE INDEX IF NOT EXISTS "version_feed_seq_idx" + ON "memory"."version" ("tenant_id", "generation", "feed_seq"); diff --git a/migrations/0007_retention.sql b/migrations/0007_retention.sql new file mode 100644 index 0000000..4366c4a --- /dev/null +++ b/migrations/0007_retention.sql @@ -0,0 +1,16 @@ +-- Retention classes + lifecycle write paths (CL-5871). +-- Orthogonal to temporal_class (ranking) and status (active/deprecated/…). + +ALTER TABLE "memory"."version" + ADD COLUMN IF NOT EXISTS "retention_class" text NOT NULL DEFAULT 'standard'; + +ALTER TABLE "memory"."version" + DROP CONSTRAINT IF EXISTS "version_retention_class_check"; + +ALTER TABLE "memory"."version" + ADD CONSTRAINT "version_retention_class_check" + CHECK ("retention_class" IN ('durable', 'standard', 'ephemeral', 'source_only')); + +CREATE INDEX IF NOT EXISTS "version_retention_ephemeral_idx" + ON "memory"."version" ("tenant_id", "retention_class", "valid_until") + WHERE "retention_class" = 'ephemeral'; diff --git a/package.json b/package.json index d754bed..963fe78 100644 --- a/package.json +++ b/package.json @@ -1,15 +1,30 @@ { "name": "@corbits/memory", "version": "0.1.2", - "description": "Mountable memory add/search/list SDK for Interchange hubs", + "description": "Mountable memory add/search/list SDK for Interchange hubs — includes resident distiller", "exports": { ".": "./src/index.ts", "./migrations": "./src/migrations.ts", "./config": "./src/mount-config.ts", - "./tools": "./src/tools/index.ts" + "./tools": "./src/tools/index.ts", + "./distiller": "./src/distiller/index.ts" }, "interchange": { - "tools": "./src/tools/index.ts" + "tools": "./src/tools/index.ts", + "grantRequirements": [ + { + "resource": "memory", + "action": "add", + "source": "tenant", + "surfaces": ["tools", "distiller", "routes"] + }, + { + "resource": "memory", + "action": "search", + "source": "tenant", + "surfaces": ["tools", "distiller", "routes"] + } + ] }, "license": "LGPL-2.1-only", "type": "module", @@ -28,6 +43,7 @@ "@intx/authz": "0.2.2", "@intx/hub-api": "0.2.2", "@intx/log": "0.2.2", + "@intx/workflow": "0.2.2", "arktype": "^2.1.29", "drizzle-orm": "^0.45.1", "hono": "^4.9.0", diff --git a/src/core/adapt-and-plan.test.ts b/src/core/adapt-and-plan.test.ts index 56def50..855b111 100644 --- a/src/core/adapt-and-plan.test.ts +++ b/src/core/adapt-and-plan.test.ts @@ -51,12 +51,12 @@ describe("adaptAndPlan", () => { it("carries entityHints and edges through from the adapted document", () => { const plan = adaptAndPlan( validAdaptedDocument({ - edges: [{ rel: "links", to: { type: "native", ref: "mail:m1" } }], + edges: [{ rel: "involves", to: { type: "native", ref: "mail:m1" } }], entityHints: [{ kind: "person", identifier: "jane@example.com" }], }), ); expect(plan.edges).toEqual([ - { rel: "links", to: { type: "native", ref: "mail:m1" } }, + { rel: "involves", to: { type: "native", ref: "mail:m1" } }, ]); expect(plan.entityHints).toEqual([ { kind: "person", identifier: "jane@example.com" }, diff --git a/src/core/corroboration.test.ts b/src/core/corroboration.test.ts new file mode 100644 index 0000000..3bcb7be --- /dev/null +++ b/src/core/corroboration.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from "bun:test"; +import { + corroborationFactor, + CORROBORATION_STRONG_FLOOR, + effectiveAuthority, + meetsStrongEvidenceGate, +} from "./corroboration.ts"; +import { + BOOST_MULTIPLIER_MAX, + BOOST_MULTIPLIER_MIN, +} from "./hybrid-search.ts"; + +describe("corroborationFactor", () => { + it("is neutral with no edges", () => { + expect(corroborationFactor({ supports: 0, contradicts: 0 })).toBe(1); + }); + + it("raises rank with independent supports", () => { + const none = corroborationFactor({ supports: 0, contradicts: 0 }); + const one = corroborationFactor({ supports: 1, contradicts: 0 }); + const many = corroborationFactor({ supports: 4, contradicts: 0 }); + expect(one).toBeGreaterThan(none); + expect(many).toBeGreaterThan(one); + expect(many).toBeLessThanOrEqual(BOOST_MULTIPLIER_MAX); + }); + + it("lowers rank with contradictions without zeroing", () => { + const base = corroborationFactor({ supports: 0, contradicts: 0 }); + const hit = corroborationFactor({ supports: 0, contradicts: 2 }); + expect(hit).toBeLessThan(base); + expect(hit).toBeGreaterThanOrEqual(BOOST_MULTIPLIER_MIN); + }); + + it("stays inside the boost envelope", () => { + for (const s of [0, 1, 2, 8, 100]) { + for (const c of [0, 1, 2, 8, 100]) { + const f = corroborationFactor({ supports: s, contradicts: c }); + expect(f).toBeGreaterThanOrEqual(BOOST_MULTIPLIER_MIN); + expect(f).toBeLessThanOrEqual(BOOST_MULTIPLIER_MAX); + } + } + }); +}); + +describe("effectiveAuthority", () => { + it("scales the capture snapshot without mutating the formula range", () => { + const snap = 0.8; + const raised = effectiveAuthority(snap, { supports: 4, contradicts: 0 }); + const lowered = effectiveAuthority(snap, { supports: 0, contradicts: 4 }); + expect(raised).toBeGreaterThan(snap * 0.99); + expect(lowered).toBeLessThan(snap); + expect(raised).toBeLessThanOrEqual(1); + expect(lowered).toBeGreaterThanOrEqual(0); + }); +}); + +describe("meetsStrongEvidenceGate", () => { + const floor = 0.3; + + it("rejects low authority even with supports", () => { + expect( + meetsStrongEvidenceGate({ + authority: 0.1, + supports: 10, + authorityFloor: floor, + }), + ).toBe(false); + }); + + it("accepts stated human above authority floor without supports", () => { + expect( + meetsStrongEvidenceGate({ + authority: 0.5, + supports: 0, + provenance: "stated", + createdByKind: "human", + authorityFloor: floor, + }), + ).toBe(true); + }); + + it("accepts corroboration at the strong floor", () => { + expect( + meetsStrongEvidenceGate({ + authority: 0.5, + supports: CORROBORATION_STRONG_FLOOR, + provenance: "inferred", + createdByKind: "agent", + authorityFloor: floor, + }), + ).toBe(true); + }); + + it("rejects inferred agent below corroboration floor", () => { + expect( + meetsStrongEvidenceGate({ + authority: 0.5, + supports: CORROBORATION_STRONG_FLOOR - 1, + provenance: "inferred", + createdByKind: "agent", + authorityFloor: floor, + }), + ).toBe(false); + }); +}); diff --git a/src/core/corroboration.ts b/src/core/corroboration.ts new file mode 100644 index 0000000..2c407bb --- /dev/null +++ b/src/core/corroboration.ts @@ -0,0 +1,88 @@ +/** + * Living claim relevancy from supports/contradicts edge counts. + * + * Capture-time `authority` stays a frozen snapshot. Search multiplies ranking + * by a bounded corroboration factor derived from graph edges (version targets). + * Core never decides claim sameness — the distiller chooses supports vs new write. + * + * See docs/RELEVANCY.md. + */ +import { + BOOST_MULTIPLIER_MAX, + BOOST_MULTIPLIER_MIN, + clampBoostMultiplier, +} from "./hybrid-search.ts"; + +/** Independent supports needed for evidence:strong (with authority floor). */ +export const CORROBORATION_STRONG_FLOOR = 2; + +/** Log-scale cap for support/contradict counts (mirrors actor-count plateau). */ +export const CORROBORATION_COUNT_LOG_CAP = 4; + +const BOOST_BASE = BOOST_MULTIPLIER_MIN; +const BOOST_SPAN = BOOST_MULTIPLIER_MAX - BOOST_MULTIPLIER_MIN; + +export type CorroborationCounts = { + supports: number; + contradicts: number; +}; + +/** + * Bounded multiplier in [0.7, 1.3]. Neutral (1.0) when no edges. + * Supports raise rank; contradicts lower it — never auto-delete. + */ +export function corroborationFactor(counts: CorroborationCounts): number { + const supports = Math.max(0, Math.floor(counts.supports)); + const contradicts = Math.max(0, Math.floor(counts.contradicts)); + if (supports === 0 && contradicts === 0) { + return 1; + } + const denom = Math.log(1 + CORROBORATION_COUNT_LOG_CAP); + const supportScore = + denom > 0 ? Math.min(1, Math.log(1 + supports) / denom) : 0; + const contradictScore = + denom > 0 ? Math.min(1, Math.log(1 + contradicts) / denom) : 0; + // Midpoint 0.5 + half support − half contradict → [0, 1] then map to envelope. + const unit = Math.min( + 1, + Math.max(0, 0.5 + 0.5 * supportScore - 0.5 * contradictScore), + ); + return clampBoostMultiplier(BOOST_BASE + BOOST_SPAN * unit); +} + +/** + * Effective authority for ranking: snapshot × corroboration factor, clamped to + * [0, 1] so authority-weighted formulas stay in range. + */ +export function effectiveAuthority( + captureAuthority: number, + counts: CorroborationCounts, +): number { + const factor = corroborationFactor(counts); + return Math.min(1, Math.max(0, captureAuthority * factor)); +} + +export type StrongEvidenceSignals = { + /** Capture-time authority (0..1). */ + authority: number; + /** Independent supports targeting this version. */ + supports: number; + provenance?: string | undefined; + createdByKind?: string | undefined; + /** Default AUTHORITY_STRONG_FLOOR from search. */ + authorityFloor: number; + corroborationFloor?: number | undefined; +}; + +/** + * Whether evidence may report strong given relevance already cleared. + * Requires authority floor AND (stated human OR supports ≥ floor). + */ +export function meetsStrongEvidenceGate(signals: StrongEvidenceSignals): boolean { + if (signals.authority < signals.authorityFloor) return false; + const floor = signals.corroborationFloor ?? CORROBORATION_STRONG_FLOOR; + const statedHuman = + signals.provenance === "stated" && signals.createdByKind === "human"; + if (statedHuman) return true; + return Math.max(0, Math.floor(signals.supports)) >= floor; +} diff --git a/src/core/embed-model-registry.test.ts b/src/core/embed-model-registry.test.ts index 19cd032..de26b05 100644 --- a/src/core/embed-model-registry.test.ts +++ b/src/core/embed-model-registry.test.ts @@ -2,15 +2,19 @@ import { describe, expect, it, mock } from "bun:test"; import type { EmbedClientConfig } from "./embed-client.ts"; import { activateEmbedModel, + activateEmbedModelByKey, + clearActiveEmbedModels, computeModelKey, cosineDistanceExpr, DimsOutOfBoundsError, discoverModelDims, EMBED_TABLE_NAME_PATTERN, embeddingTableName, + ensureEmbedModel, type EmbedRegistrySqlClient, HALFVEC_INDEX_MAX_DIMS, resolveActiveEmbedTable, + resolveEmbedTableByModelKey, VECTOR_INDEX_MAX_DIMS, } from "./embed-model-registry.ts"; @@ -272,6 +276,131 @@ describe("resolveActiveEmbedTable", () => { tableName: `"memory"."embedding_${modelKey}"`, dims: 768, modelId: baseConfig.modelId, + modelKey, }); }); }); + +describe("ensureEmbedModel", () => { + it("inserts with status='ready' (never active) so live dense search is untouched", async () => { + const { client, queries } = createMockClient(); + await ensureEmbedModel(client, "tenant-1", baseConfig, fixtureFetch(768)); + + const insertQuery = queries.find((q) => + q.sql.includes('INSERT INTO "memory"."embed_model"'), + ); + expect(insertQuery?.sql).toContain("'ready'"); + expect(insertQuery?.sql).not.toContain("'active'"); + + // No UPDATE ... SET status='active' issued by ensure. + expect( + queries.some((q) => q.sql.includes("SET status = 'active'")), + ).toBe(false); + }); + + it("still creates the per-model table + indexes (table usable on replay)", async () => { + const { client, queries } = createMockClient(); + const result = await ensureEmbedModel(client, "tenant-1", baseConfig, fixtureFetch(768)); + const createTableQuery = queries.find((q) => q.sql.includes("CREATE TABLE IF NOT EXISTS")); + expect(createTableQuery?.sql).toContain(result.tableName); + expect(queries.some((q) => q.sql.includes("USING hnsw"))).toBe(true); + }); +}); + +describe("activateEmbedModel (split)", () => { + it("ensures then issues UPDATE ... SET status='active', updated_at=now()", async () => { + const { client, queries } = createMockClient(); + await activateEmbedModel(client, "tenant-1", baseConfig, fixtureFetch(768)); + + const updateQuery = queries.find((q) => + q.sql.includes("SET status = 'active'"), + ); + expect(updateQuery).toBeDefined(); + expect(updateQuery?.sql).toContain("updated_at = now()"); + }); + + it("ensure then activate does not leave only-ready when activate follows", async () => { + const { client, queries } = createMockClient(); + await ensureEmbedModel(client, "tenant-1", baseConfig, fixtureFetch(768)); + const ensureQueries = queries.length; + await activateEmbedModel(client, "tenant-1", baseConfig, fixtureFetch(768)); + // activate always issues the exclusive active UPDATE after ensure work + expect( + queries.slice(ensureQueries).some((q) => q.sql.includes("SET status = 'active'")), + ).toBe(true); + expect( + queries.slice(ensureQueries).some((q) => q.sql.includes("SET status = 'ready'")), + ).toBe(true); + }); +}); + +describe("clearActiveEmbedModels", () => { + it("demotes every active row for the tenant to ready", async () => { + const { client, queries } = createMockClient(); + await clearActiveEmbedModels(client, "tenant-1"); + expect(queries).toHaveLength(1); + expect(queries[0]?.sql).toContain("SET status = 'ready'"); + expect(queries[0]?.sql).toContain("status = 'active'"); + expect(queries[0]?.params).toEqual(["tenant-1"]); + }); +}); + +describe("activateEmbedModelByKey", () => { + it("selects by model_key then exclusives-activates without probing embed", async () => { + const modelKey = "abcdef0123456789"; + const queries: Array<{ sql: string; params: readonly unknown[] }> = []; + const client: EmbedRegistrySqlClient = { + query: (sql, params) => { + queries.push({ sql, params }); + return Promise.resolve([ + { model_key: modelKey, model_id: "text-embed-3", dims: 768 }, + ]); + }, + }; + const result = await activateEmbedModelByKey(client, "tenant-1", modelKey); + expect(result.modelKey).toBe(modelKey); + expect(result.dims).toBe(768); + expect(queries[0]?.sql).toContain("SELECT model_key"); + expect(queries.some((q) => q.sql.includes("SET status = 'ready'"))).toBe(true); + expect(queries.some((q) => q.sql.includes("SET status = 'active'"))).toBe(true); + }); + + it("throws when the registry row is missing", async () => { + const client: EmbedRegistrySqlClient = { + query: () => Promise.resolve([]), + }; + await expect( + activateEmbedModelByKey(client, "tenant-1", "abcdef0123456789"), + ).rejects.toThrow(/no embed_model row/); + }); + + it("rejects an invalid model_key format", async () => { + const client: EmbedRegistrySqlClient = { query: () => Promise.resolve([]) }; + await expect( + activateEmbedModelByKey(client, "tenant-1", "not-a-key"), + ).rejects.toThrow(/invalid modelKey/); + }); +}); + +describe("resolveEmbedTableByModelKey", () => { + it("returns null when no row for this model_key", async () => { + const client: EmbedRegistrySqlClient = { query: () => Promise.resolve([]) }; + expect(await resolveEmbedTableByModelKey(client, "tenant-1", "abcdef0123456789")).toBeNull(); + }); + + it("returns the table info regardless of status (ready or active)", async () => { + const modelKey = "abcdef0123456789"; + const client: EmbedRegistrySqlClient = { + query: () => + Promise.resolve([{ model_key: modelKey, model_id: "m", dims: 512 }]), + }; + const result = await resolveEmbedTableByModelKey(client, "tenant-1", modelKey); + expect(result?.dims).toBe(512); + expect(result?.tableName).toBe(`"memory"."embedding_${modelKey}"`); + }); + + it("rejects an invalid model_key format", async () => { + const client: EmbedRegistrySqlClient = { query: () => Promise.resolve([]) }; + expect(() => resolveEmbedTableByModelKey(client, "tenant-1", "not-a-key")).toThrow(); + }); +}); diff --git a/src/core/embed-model-registry.ts b/src/core/embed-model-registry.ts index 147e125..10a04a0 100644 --- a/src/core/embed-model-registry.ts +++ b/src/core/embed-model-registry.ts @@ -111,7 +111,15 @@ export interface ActivateEmbedModelResult { modelKey: string; } -export async function activateEmbedModel( +/** + * Ensure the per-model embedding table and registry row exist without making + * this model the tenant's active dense-search target. + * + * Used by transform/replay so a staged embed model flip never steals + * `resolveActiveEmbedTable` from live. Idempotent: CREATE TABLE IF NOT EXISTS + * + registry upsert that never promotes status to `active`. + */ +export async function ensureEmbedModel( client: EmbedRegistrySqlClient, tenantId: string, config: EmbedClientConfig, @@ -122,11 +130,14 @@ export async function activateEmbedModel( const tableName = embeddingTableName(modelKey); const bare = embeddingTableBareName(modelKey); + // status stays 'ready' on insert; ON CONFLICT never overwrites an existing + // status (so an already-active model remains active) and never bumps + // updated_at (so a ready model cannot leapfrog resolveActiveEmbedTable). await client.query( `INSERT INTO "memory"."embed_model" (id, tenant_id, model_key, model_id, dims, status, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, 'active', now(), now()) + VALUES ($1, $2, $3, $4, $5, 'ready', now(), now()) ON CONFLICT (tenant_id, model_key) - DO UPDATE SET model_id = EXCLUDED.model_id, dims = EXCLUDED.dims, updated_at = now()`, + DO UPDATE SET model_id = EXCLUDED.model_id, dims = EXCLUDED.dims`, [randomUUID(), tenantId, modelKey, config.modelId, dims], ); @@ -148,7 +159,7 @@ export async function activateEmbedModel( ); // Composite over bare tenant_id: the read path filters tenant_id plus a // chunk_id set; this is a B-tree membership filter (fetchChunkVectors also - // selects embedding). Runs on every activation so a pre-FK table still gets + // selects embedding). Runs on every ensure so a pre-FK table still gets // the index. await client.query( `CREATE INDEX IF NOT EXISTS ${bare}_tenant_chunk_idx ON ${tableName} (tenant_id, chunk_id)`, @@ -178,7 +189,7 @@ export async function activateEmbedModel( // cosineDistanceExpr emits for these dims, or the planner ignores the // index. No ivfflat fallback on this path: halfvec requires // pgvector >= 0.7.0 and every such release has hnsw, so a failure here - // means the extension is too old for halfvec at all — let activation + // means the extension is too old for halfvec at all — let ensure // fail loudly at this boundary rather than accept an unindexable model. // IF NOT EXISTS also retrofits the index onto a table created before // halfvec support existed; on a large populated table this build can @@ -192,10 +203,29 @@ export async function activateEmbedModel( return { tableName, dims, modelId: config.modelId, modelKey }; } +/** + * Ensure the table exists, then promote this model to the tenant's active + * dense-search target (`status='active'`, `updated_at=now()`). Other active + * rows for the tenant are demoted to `ready` so `resolveActiveEmbedTable` + * cannot pick a stale peer by updated_at race. Live capture uses this path; + * transform/replay must use `ensureEmbedModel` only. + */ +export async function activateEmbedModel( + client: EmbedRegistrySqlClient, + tenantId: string, + config: EmbedClientConfig, + fetchImpl: typeof fetch = fetch, +): Promise { + const result = await ensureEmbedModel(client, tenantId, config, fetchImpl); + await setActiveEmbedModelExclusive(client, tenantId, result.modelKey); + return result; +} + export interface ActiveEmbedTable { tableName: string; dims: number; modelId: string; + modelKey?: string; } export async function resolveActiveEmbedTable( @@ -217,5 +247,112 @@ export async function resolveActiveEmbedTable( tableName: embeddingTableName(modelKey), dims: row.dims as number, modelId: row.model_id as string, + modelKey, + }; +} + +/** + * Promote an already-registered embed model to the tenant's active dense + * target by `model_key` only (no embed endpoint probe). Used by demote to + * restore the pre-promote live model without needing baseUrl/modelId. + * + * Throws if the registry row is missing — demote must not silently leave + * dense search on the promoted model. Demotes other active rows for the + * tenant to `ready`. + */ +export async function activateEmbedModelByKey( + client: EmbedRegistrySqlClient, + tenantId: string, + modelKey: string, +): Promise { + if (!/^[a-f0-9]{16}$/.test(modelKey)) { + throw new Error(`activateEmbedModelByKey: invalid modelKey "${modelKey}"`); + } + const rows = await client.query( + `SELECT model_key, model_id, dims FROM "memory"."embed_model" + WHERE tenant_id = $1 AND model_key = $2`, + [tenantId, modelKey], + ); + const row = rows[0]; + if (!row) { + throw new Error( + `activateEmbedModelByKey: no embed_model row for tenant=${tenantId} model_key=${modelKey}`, + ); + } + await setActiveEmbedModelExclusive(client, tenantId, modelKey); + return { + tableName: embeddingTableName(modelKey), + dims: row.dims as number, + modelId: row.model_id as string, + modelKey, + }; +} + +/** + * Clear every active embed model for the tenant (status → ready). Used when + * demoting a promote that had no prior live model so dense search degrades + * cleanly instead of remaining on the promoted table after the corpus swap. + */ +export async function clearActiveEmbedModels( + client: EmbedRegistrySqlClient, + tenantId: string, +): Promise { + await client.query( + `UPDATE "memory"."embed_model" + SET status = 'ready', updated_at = now() + WHERE tenant_id = $1 AND status = 'active'`, + [tenantId], + ); +} + +/** Make `modelKey` the sole active model for the tenant (single statement). */ +async function setActiveEmbedModelExclusive( + client: EmbedRegistrySqlClient, + tenantId: string, + modelKey: string, +): Promise { + // One round-trip: demote every other active row and activate the target. + // Concurrent activate still races at the app level; this removes the + // two-UPDATE window where zero or two actives can briefly exist. + await client.query( + `WITH demoted AS ( + UPDATE "memory"."embed_model" + SET status = 'ready', updated_at = now() + WHERE tenant_id = $1 AND status = 'active' AND model_key <> $2 + RETURNING 1 + ) + UPDATE "memory"."embed_model" + SET status = 'active', updated_at = now() + WHERE tenant_id = $1 AND model_key = $2`, + [tenantId, modelKey], + ); +} + +/** + * Resolve any registered embed table by model_key (active or ready). + * Used when searching a non-live generation whose transform_config embeds + * with a model that is not the tenant's active one. + */ +export async function resolveEmbedTableByModelKey( + client: EmbedRegistrySqlClient, + tenantId: string, + modelKey: string, +): Promise { + if (!/^[a-f0-9]{16}$/.test(modelKey)) { + throw new Error(`resolveEmbedTableByModelKey: invalid modelKey "${modelKey}"`); + } + const rows = await client.query( + `SELECT model_key, model_id, dims FROM "memory"."embed_model" + WHERE tenant_id = $1 AND model_key = $2 + LIMIT 1`, + [tenantId, modelKey], + ); + const row = rows[0]; + if (!row) return null; + return { + tableName: embeddingTableName(modelKey), + dims: row.dims as number, + modelId: row.model_id as string, + modelKey, }; } diff --git a/src/core/embed-sql.ts b/src/core/embed-sql.ts index 76a87a3..10da901 100644 --- a/src/core/embed-sql.ts +++ b/src/core/embed-sql.ts @@ -1,4 +1,4 @@ -import type { RawSql } from "../db/client.ts"; +import type postgres from "postgres"; import type { EmbedRegistrySqlClient } from "./embed-model-registry.ts"; // Generic bridge from the engine's postgres-js handle to the minimal @@ -7,8 +7,13 @@ import type { EmbedRegistrySqlClient } from "./embed-model-registry.ts"; // Positional `$1`/`$2` placeholders via `sql.unsafe`. Despite the historical // name this is not embed-specific — `EmbedRegistrySqlClient` and // `FtsVerifySqlClient` (core/fts-language.ts) are structurally the same -// shape, so one bridge serves both. -export function createRawSqlClient(sql: RawSql): EmbedRegistrySqlClient { +// shape, so one bridge serves both. Accepts either the top-level connection +// or a `sql.begin()` transaction handle, so callers can fold registry writes +// into a larger transaction (see transform.ts promote/demote) instead of +// committing them separately. +export function createRawSqlClient( + sql: postgres.Sql<{}> | postgres.TransactionSql<{}>, +): EmbedRegistrySqlClient { return { async query(sqlText, params) { const rows = await sql.unsafe(sqlText, [...params] as never[]); diff --git a/src/core/enums.lockstep.test.ts b/src/core/enums.lockstep.test.ts new file mode 100644 index 0000000..6c3708e --- /dev/null +++ b/src/core/enums.lockstep.test.ts @@ -0,0 +1,144 @@ +import { describe, expect, it } from "bun:test"; +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { + EDGE_RELS, + EDGE_REF_TYPES_DB, + LINEAGE_CLASSES, + PROVENANCE_MODES, + RETENTION_CLASSES, + TEMPORAL_CLASSES, +} from "./enums.ts"; +import { MemoryEdgeRelSchema, MemoryEdgeRefTypeSchema } from "./schemas/entity-edge.ts"; +import { + LineageClassSchema, + ProvenanceModeSchema, + RetentionClassSchema, + TemporalClassSchema, +} from "./schemas/document.ts"; +import { type } from "arktype"; + +const MIGRATIONS_DIR = join(import.meta.dir, "../../migrations"); + +function allMigrationSql(): string { + const files = readdirSync(MIGRATIONS_DIR) + .filter((f) => f.endsWith(".sql")) + .sort(); + return files + .map((f) => readFileSync(join(MIGRATIONS_DIR, f), "utf8")) + .join("\n"); +} + +/** Pull the last CHECK (... IN (...)) body for a named constraint. */ +function lastCheckInList(sql: string, constraintName: string): string[] { + const re = new RegExp( + `CONSTRAINT\\s+"${constraintName}"\\s+CHECK\\s*\\(\\s*"[^"]+"\\s+IN\\s*\\(([\\s\\S]*?)\\)\\s*\\)`, + "gi", + ); + let match: RegExpExecArray | null; + let last: string | null = null; + while ((match = re.exec(sql)) !== null) { + last = match[1] ?? null; + } + if (last === null) { + throw new Error(`constraint ${constraintName} not found in migrations`); + } + return [...last.matchAll(/'([^']+)'/g)].map((m) => m[1] as string); +} + +function sorted(values: readonly string[]): string[] { + return [...values].sort(); +} + +describe("enum lockstep: TS constants match migration CHECK constraints", () => { + const sql = allMigrationSql(); + + it("edge_rel_check matches EDGE_RELS", () => { + expect(sorted(lastCheckInList(sql, "edge_rel_check"))).toEqual( + sorted(EDGE_RELS), + ); + }); + + it("edge_from_type_check matches EDGE_REF_TYPES_DB", () => { + expect(sorted(lastCheckInList(sql, "edge_from_type_check"))).toEqual( + sorted(EDGE_REF_TYPES_DB), + ); + }); + + it("edge_to_type_check matches EDGE_REF_TYPES_DB", () => { + expect(sorted(lastCheckInList(sql, "edge_to_type_check"))).toEqual( + sorted(EDGE_REF_TYPES_DB), + ); + }); + + it("version_source_class_check matches LINEAGE_CLASSES", () => { + expect(sorted(lastCheckInList(sql, "version_source_class_check"))).toEqual( + sorted(LINEAGE_CLASSES), + ); + }); + + it("version_provenance_check matches PROVENANCE_MODES", () => { + expect(sorted(lastCheckInList(sql, "version_provenance_check"))).toEqual( + sorted(PROVENANCE_MODES), + ); + }); + + it("version_temporal_class_check matches TEMPORAL_CLASSES", () => { + expect(sorted(lastCheckInList(sql, "version_temporal_class_check"))).toEqual( + sorted(TEMPORAL_CLASSES), + ); + }); + + it("version_retention_class_check matches RETENTION_CLASSES", () => { + expect(sorted(lastCheckInList(sql, "version_retention_class_check"))).toEqual( + sorted(RETENTION_CLASSES), + ); + }); +}); + +describe("enum lockstep: arktype accepts every SSOT value and rejects unknown", () => { + it("MemoryEdgeRelSchema accepts all EDGE_RELS", () => { + for (const rel of EDGE_RELS) { + const out = MemoryEdgeRelSchema(rel); + expect(out instanceof type.errors ? out.summary : out).toBe(rel); + } + expect(MemoryEdgeRelSchema("produced_by") instanceof type.errors).toBe( + true, + ); + }); + + it("MemoryEdgeRefTypeSchema accepts adapter set including native", () => { + for (const t of ["document", "version", "chunk", "entity", "native"] as const) { + const out = MemoryEdgeRefTypeSchema(t); + expect(out instanceof type.errors ? out.summary : out).toBe(t); + } + }); + + it("LineageClassSchema accepts LINEAGE_CLASSES only", () => { + for (const c of LINEAGE_CLASSES) { + expect(LineageClassSchema(c) instanceof type.errors).toBe(false); + } + expect(LineageClassSchema("thread") instanceof type.errors).toBe(true); + }); + + it("ProvenanceModeSchema accepts PROVENANCE_MODES only", () => { + for (const p of PROVENANCE_MODES) { + expect(ProvenanceModeSchema(p) instanceof type.errors).toBe(false); + } + expect(ProvenanceModeSchema("guessed") instanceof type.errors).toBe(true); + }); + + it("TemporalClassSchema accepts TEMPORAL_CLASSES only", () => { + for (const t of TEMPORAL_CLASSES) { + expect(TemporalClassSchema(t) instanceof type.errors).toBe(false); + } + expect(TemporalClassSchema("forecast") instanceof type.errors).toBe(true); + }); + + it("RetentionClassSchema accepts RETENTION_CLASSES only", () => { + for (const r of RETENTION_CLASSES) { + expect(RetentionClassSchema(r) instanceof type.errors).toBe(false); + } + expect(RetentionClassSchema("forever") instanceof type.errors).toBe(true); + }); +}); diff --git a/src/core/enums.ts b/src/core/enums.ts new file mode 100644 index 0000000..dc2399c --- /dev/null +++ b/src/core/enums.ts @@ -0,0 +1,82 @@ +// Single source of truth for memory-plane enums that also appear as +// Postgres CHECK constraints. Arktype schemas import these; the lockstep +// test asserts the latest migration SQL matches exactly. + +/** Graph edge relationship kinds stored on memory.edge.rel. */ +export const EDGE_RELS = [ + "mentions", + "about", + "authored_by", + "involves", + "part_of", + "derived_from", + "supports", + "contradicts", + "supersedes", +] as const; +export type EdgeRel = (typeof EDGE_RELS)[number]; + +/** + * Endpoint kinds that may be written to memory.edge.from_type / to_type. + * Adapter-facing hints may also use `native` (see EDGE_REF_TYPES_ADAPTER); + * capture resolves native → entity before insert. + */ +export const EDGE_REF_TYPES_DB = [ + "document", + "version", + "chunk", + "entity", +] as const; +export type EdgeRefTypeDb = (typeof EDGE_REF_TYPES_DB)[number]; + +/** Adapter-facing edge endpoint kinds, including planning-time `native`. */ +export const EDGE_REF_TYPES_ADAPTER = [ + ...EDGE_REF_TYPES_DB, + "native", +] as const; +export type EdgeRefTypeAdapter = (typeof EDGE_REF_TYPES_ADAPTER)[number]; + +/** + * Data-lineage class stored on memory.version.source_class. + * Orthogonal to AuthoritySourceClass (ranking priors: thread/channel/…). + */ +export const LINEAGE_CLASSES = ["native", "imported", "derived"] as const; +export type LineageClass = (typeof LINEAGE_CLASSES)[number]; + +/** + * How the version's content was obtained relative to assertion. + * Orthogonal to created_by_kind (who) and lineageClass (where from). + */ +export const PROVENANCE_MODES = ["stated", "inferred", "unknown"] as const; +export type ProvenanceMode = (typeof PROVENANCE_MODES)[number]; + +/** + * Temporal ranking class stored on memory.version.temporal_class. + * See docs/TEMPORAL.md. + */ +export const TEMPORAL_CLASSES = [ + "event", + "deadline", + "state", + "lesson", +] as const; +export type TemporalClass = (typeof TEMPORAL_CLASSES)[number]; + +/** + * Retention class on memory.version.retention_class (CL-5871). + * Orthogonal to temporal_class (ranking) and status (lifecycle). + */ +export const RETENTION_CLASSES = [ + "durable", + "standard", + "ephemeral", + "source_only", +] as const; +export type RetentionClass = (typeof RETENTION_CLASSES)[number]; + +/** Build an arktype union string from a const string array. */ +export function arktypeStringUnion( + values: readonly string[], +): string { + return values.map((v) => `'${v}'`).join("|"); +} diff --git a/src/core/hybrid-search.test.ts b/src/core/hybrid-search.test.ts index 7637dac..c2beb12 100644 --- a/src/core/hybrid-search.test.ts +++ b/src/core/hybrid-search.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test"; import { BOOST_MULTIPLIER_MAX, BOOST_MULTIPLIER_MIN, + DEADLINE_LOOKAHEAD_MS, MAX_BATCH_QUERIES, RECENCY_HALF_LIFE_MS, authorityBoostMultiplier, @@ -11,6 +12,7 @@ import { isBatchQueriesWithinBound, normalizeScoresToUnit, recencyBoostMultiplier, + temporalRecencyMultiplier, toRankedCandidates, } from "./hybrid-search.ts"; @@ -194,3 +196,80 @@ describe("recencyBoostMultiplier", () => { ); }); }); + +describe("temporalRecencyMultiplier", () => { + const now = new Date("2026-07-20T00:00:00.000Z"); + + test("event matches recencyBoostMultiplier", () => { + const oneHalfLifeAgo = new Date(now.getTime() - RECENCY_HALF_LIFE_MS); + expect( + temporalRecencyMultiplier({ + temporalClass: "event", + occurredAt: oneHalfLifeAgo, + validUntil: null, + now, + }), + ).toBeCloseTo(recencyBoostMultiplier(oneHalfLifeAgo, now), 10); + }); + + test("state and lesson are neutral regardless of age", () => { + const old = new Date("2020-01-01T00:00:00.000Z"); + for (const temporalClass of ["state", "lesson"] as const) { + expect( + temporalRecencyMultiplier({ + temporalClass, + occurredAt: old, + validUntil: null, + now, + }), + ).toBe(1.0); + } + }); + + test("deadline far out is neutral", () => { + const far = new Date(now.getTime() + DEADLINE_LOOKAHEAD_MS * 2); + expect( + temporalRecencyMultiplier({ + temporalClass: "deadline", + occurredAt: now, + validUntil: far, + now, + }), + ).toBe(1.0); + }); + + test("deadline at expiry approaches the urgency ceiling (~1.3)", () => { + const almostDue = new Date(now.getTime() + 1); + const mult = temporalRecencyMultiplier({ + temporalClass: "deadline", + occurredAt: now, + validUntil: almostDue, + now, + }); + expect(mult).toBeGreaterThan(1.25); + expect(mult).toBeLessThanOrEqual(BOOST_MULTIPLIER_MAX); + }); + + test("expired deadline falls to the boost floor (still retrievable)", () => { + const past = new Date(now.getTime() - 1000); + expect( + temporalRecencyMultiplier({ + temporalClass: "deadline", + occurredAt: now, + validUntil: past, + now, + }), + ).toBe(BOOST_MULTIPLIER_MIN); + }); + + test("deadline with null validUntil is neutral", () => { + expect( + temporalRecencyMultiplier({ + temporalClass: "deadline", + occurredAt: now, + validUntil: null, + now, + }), + ).toBe(1.0); + }); +}); diff --git a/src/core/hybrid-search.ts b/src/core/hybrid-search.ts index 76a0579..4aeda1d 100644 --- a/src/core/hybrid-search.ts +++ b/src/core/hybrid-search.ts @@ -155,3 +155,41 @@ export function recencyBoostMultiplier( const decay = Math.pow(2, -ageMs / halfLifeMs); return clampBoostMultiplier(BOOST_BASE + BOOST_SPAN * decay); } + +// How far ahead of valid_until a deadline starts ramping urgency (neutral +// before this window; approaches the boost ceiling at the deadline). +export const DEADLINE_LOOKAHEAD_MS = 7 * 24 * 60 * 60 * 1000; + +export type TemporalRecencyInput = { + temporalClass: "event" | "deadline" | "state" | "lesson"; + occurredAt: Date; + validUntil: Date | null; + now: Date; + halfLifeMs?: number; +}; + +/** + * Recency prior by temporal class (docs/TEMPORAL.md): + * - event: exponential decay from occurred_at (existing half-life) + * - deadline: neutral far out; urgency ramp in lookahead before valid_until; + * floor after expiry (still history-retrievable, not deleted) + * - state / lesson: no decay while active (superseded rows are status-filtered) + */ +export function temporalRecencyMultiplier(input: TemporalRecencyInput): number { + const halfLifeMs = input.halfLifeMs ?? RECENCY_HALF_LIFE_MS; + switch (input.temporalClass) { + case "event": + return recencyBoostMultiplier(input.occurredAt, input.now, halfLifeMs); + case "state": + case "lesson": + return 1.0; + case "deadline": { + if (input.validUntil === null) return 1.0; + const remaining = input.validUntil.getTime() - input.now.getTime(); + if (remaining <= 0) return BOOST_MULTIPLIER_MIN; + if (remaining >= DEADLINE_LOOKAHEAD_MS) return 1.0; + const urgency = 1 - remaining / DEADLINE_LOOKAHEAD_MS; + return clampBoostMultiplier(1.0 + BOOST_SPAN * urgency * 0.5); + } + } +} diff --git a/src/core/schemas/adapted-document.test.ts b/src/core/schemas/adapted-document.test.ts index 7c75b01..d059c92 100644 --- a/src/core/schemas/adapted-document.test.ts +++ b/src/core/schemas/adapted-document.test.ts @@ -15,7 +15,7 @@ describe("AdaptedDocumentSchema", () => { entityHints: [{ kind: "person", identifier: "jane@example.com" }], edges: [ { rel: "about", to: { type: "entity", ref: "acme-co" } }, - { rel: "produced_by", to: { type: "native", ref: "principal_1" } }, + { rel: "authored_by", to: { type: "native", ref: "principal_1" } }, ], chunks: [ { ordinal: 0, text: "Opening remarks.", role: "summary" }, diff --git a/src/core/schemas/adapted-document.ts b/src/core/schemas/adapted-document.ts index 2dd5d3c..1df3256 100644 --- a/src/core/schemas/adapted-document.ts +++ b/src/core/schemas/adapted-document.ts @@ -1,5 +1,10 @@ import { type } from "arktype"; -import { CreatedByKindSchema } from "./document.ts"; +import { + CreatedByKindSchema, + LineageClassSchema, + ProvenanceModeSchema, + TemporalClassSchema, +} from "./document.ts"; import { MemoryEdgeHintSchema } from "./entity-edge.ts"; import { AuthoritySourceClassSchema } from "../authority.ts"; @@ -48,6 +53,14 @@ export type RawPointer = typeof RawPointerSchema.infer; // // Document access is grant tags only (`accessTags`) — the security boundary // (docs/AUTHZ-DOCUMENT-ACCESS.md). +// +// Two orthogonal "class" axes on the write path: +// - sourceClass: ranking prior (thread/channel/call/record/native) for +// computeAuthority — never written to memory.version.source_class. +// - lineageClass: data-lineage stored on memory.version.source_class +// (native|imported|derived). Defaults to native at capture. +// provenance: how the content was obtained (stated|inferred|unknown); +// defaults to stated at capture for human/adapter paths. export const AdaptedDocumentSchema = type({ kind: `1 <= string <= ${MAX_KIND_CHARS}`, title: `1 <= string <= ${MAX_TITLE_CHARS}`, @@ -67,6 +80,11 @@ export const AdaptedDocumentSchema = type({ "actorCount?": "number", "sourceClass?": AuthoritySourceClassSchema, "hasSocialSignal?": "boolean", + "lineageClass?": LineageClassSchema, + "provenance?": ProvenanceModeSchema, + "temporalClass?": TemporalClassSchema, + "validFrom?": "string", + "validUntil?": "string", contentHash: "string", }); export type AdaptedDocument = typeof AdaptedDocumentSchema.infer; diff --git a/src/core/schemas/claim-bearing.test.ts b/src/core/schemas/claim-bearing.test.ts new file mode 100644 index 0000000..3c02743 --- /dev/null +++ b/src/core/schemas/claim-bearing.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from "bun:test"; +import { type } from "arktype"; +import { AdaptedDocumentSchema } from "./adapted-document.ts"; +import { MemoryEdgeHintSchema } from "./entity-edge.ts"; +import type { AdaptedDocument } from "./adapted-document.ts"; + +/** + * Claim-bearing write shape: a distilled claim is a normal AdaptedDocument + * with inferred provenance, derived lineage, and a derived_from edge. + * Core never runs inference — it only accepts this shape. + */ +describe("claim-bearing AdaptedDocument shape", () => { + it("accepts a derived claim with provenance, lineage, and derived_from", () => { + const claim: AdaptedDocument = { + kind: "claim", + title: "Acme renews in Q3", + externalRef: "claim:acme-q3-renewal", + accessTags: ["memory.tenant:t1"], + entityHints: [{ kind: "org", identifier: "acme.com" }], + edges: [ + { + rel: "derived_from", + to: { type: "version", ref: "kver_source_1" }, + }, + { + rel: "supports", + to: { type: "version", ref: "kver_prior_claim" }, + }, + ], + chunks: [ + { + ordinal: 0, + text: "Acme is expected to renew in Q3 at a 12% expansion.", + }, + ], + actor: { kind: "agent", agentId: "distiller-v1" }, + sourceClass: "record", + lineageClass: "derived", + provenance: "inferred", + contentHash: "sha256:claim-1", + }; + const out = AdaptedDocumentSchema(claim); + expect(out instanceof type.errors ? out.summary : out).toEqual(claim); + }); + + it("accepts every claim-bearing edge rel", () => { + for (const rel of [ + "derived_from", + "supports", + "contradicts", + "supersedes", + "authored_by", + "involves", + "part_of", + ] as const) { + const out = MemoryEdgeHintSchema({ + rel, + to: { type: "version", ref: "kver_1" }, + }); + expect(out instanceof type.errors).toBe(false); + } + }); + + it("rejects pre-claim edge rels that were never DB-valid", () => { + for (const rel of ["produced_by", "links", "parent", "waiting_on"]) { + const out = MemoryEdgeHintSchema({ + rel, + to: { type: "entity", ref: "e1" }, + }); + expect(out instanceof type.errors).toBe(true); + } + }); + + it("keeps ranking sourceClass independent of lineageClass", () => { + const out = AdaptedDocumentSchema({ + kind: "call_transcript", + title: "Call", + externalRef: "call:1", + accessTags: ["memory.tenant:t1"], + entityHints: [], + chunks: [{ ordinal: 0, text: "hi" }], + contentHash: "sha256:x", + sourceClass: "channel", + lineageClass: "native", + provenance: "stated", + }); + expect(out instanceof type.errors).toBe(false); + if (!(out instanceof type.errors)) { + expect(out.sourceClass).toBe("channel"); + expect(out.lineageClass).toBe("native"); + } + }); +}); diff --git a/src/core/schemas/document.test.ts b/src/core/schemas/document.test.ts index 4645578..6111b40 100644 --- a/src/core/schemas/document.test.ts +++ b/src/core/schemas/document.test.ts @@ -57,6 +57,10 @@ describe("MemoryVersionSchema", () => { deprecated_reason: null, created_by_principal_id: "principal_1", created_by_kind: "human", + provenance: "stated", + source_class: "native", + temporal_class: "event", + retention_class: "standard", }; const out = MemoryVersionSchema(fixture); expect(out instanceof type.errors ? out.summary : out).toEqual(fixture); @@ -78,6 +82,9 @@ describe("MemoryVersionSchema", () => { deprecated_reason: null, created_by_principal_id: null, created_by_kind: "human", + provenance: "stated", + source_class: "native", + temporal_class: "event", }); expect(out instanceof type.errors).toBe(true); }); diff --git a/src/core/schemas/document.ts b/src/core/schemas/document.ts index 4a7725b..5be9c89 100644 --- a/src/core/schemas/document.ts +++ b/src/core/schemas/document.ts @@ -1,4 +1,11 @@ import { type } from "arktype"; +import { + LINEAGE_CLASSES, + PROVENANCE_MODES, + RETENTION_CLASSES, + TEMPORAL_CLASSES, + arktypeStringUnion, +} from "../enums.ts"; export const MemoryVersionStatusSchema = type( "'active'|'superseded'|'deprecated'|'archived'|'tombstoned'", @@ -8,6 +15,28 @@ export type MemoryVersionStatus = typeof MemoryVersionStatusSchema.infer; export const CreatedByKindSchema = type("'human'|'agent'|'system'|'adapter'"); export type CreatedByKind = typeof CreatedByKindSchema.infer; +export const LineageClassSchema = type( + arktypeStringUnion(LINEAGE_CLASSES) as "'native'|'imported'|'derived'", +); +export type LineageClass = typeof LineageClassSchema.infer; + +export const ProvenanceModeSchema = type( + arktypeStringUnion(PROVENANCE_MODES) as "'stated'|'inferred'|'unknown'", +); +export type ProvenanceMode = typeof ProvenanceModeSchema.infer; + +export const TemporalClassSchema = type( + arktypeStringUnion(TEMPORAL_CLASSES) as + "'event'|'deadline'|'state'|'lesson'", +); +export type TemporalClass = typeof TemporalClassSchema.infer; + +export const RetentionClassSchema = type( + arktypeStringUnion(RETENTION_CLASSES) as + "'durable'|'standard'|'ephemeral'|'source_only'", +); +export type RetentionClass = typeof RetentionClassSchema.infer; + // The stable logical row for a captured source, deduped on (tenant_id, // adapter, external_ref). Document access is grant tags only. export const MemoryDocumentSchema = type({ @@ -26,6 +55,9 @@ export type MemoryDocument = typeof MemoryDocumentSchema.infer; // The versioned body of a document. Chunks belong to a version_id, never // reused across versions. +// occurred_at is effective time the content refers to (event time / state +// effective time / deadline establishment). ingested_at is when the plane +// learned it. Validity window is optional (deadline/state claims). export const MemoryVersionSchema = type({ id: "string", tenant_id: "string", @@ -42,5 +74,11 @@ export const MemoryVersionSchema = type({ created_by_principal_id: "string | null", created_by_kind: CreatedByKindSchema, "generator_agent_id?": "string", + provenance: ProvenanceModeSchema, + source_class: LineageClassSchema, + temporal_class: TemporalClassSchema, + retention_class: RetentionClassSchema, + "valid_from?": "string | null", + "valid_until?": "string | null", }); export type MemoryVersion = typeof MemoryVersionSchema.infer; diff --git a/src/core/schemas/entity-edge.test.ts b/src/core/schemas/entity-edge.test.ts index 7635b4b..439f158 100644 --- a/src/core/schemas/entity-edge.test.ts +++ b/src/core/schemas/entity-edge.test.ts @@ -52,7 +52,7 @@ describe("MemoryEdgeSchema", () => { describe("MemoryEdgeHintSchema", () => { it("parses a full fixture", () => { const out = MemoryEdgeHintSchema({ - rel: "produced_by", + rel: "authored_by", to: { type: "native", ref: "principal_1" }, }); expect(out instanceof type.errors).toBe(false); @@ -60,9 +60,19 @@ describe("MemoryEdgeHintSchema", () => { it("rejects a hint whose to is missing ref", () => { const out = MemoryEdgeHintSchema({ - rel: "produced_by", + rel: "authored_by", to: { type: "native" }, }); expect(out instanceof type.errors).toBe(true); }); + + it("accepts version and chunk endpoint types", () => { + for (const endpointType of ["version", "chunk"] as const) { + const out = MemoryEdgeHintSchema({ + rel: "derived_from", + to: { type: endpointType, ref: "id_1" }, + }); + expect(out instanceof type.errors).toBe(false); + } + }); }); diff --git a/src/core/schemas/entity-edge.ts b/src/core/schemas/entity-edge.ts index ff12c21..cd391f4 100644 --- a/src/core/schemas/entity-edge.ts +++ b/src/core/schemas/entity-edge.ts @@ -1,4 +1,9 @@ import { type } from "arktype"; +import { + EDGE_RELS, + EDGE_REF_TYPES_ADAPTER, + arktypeStringUnion, +} from "../enums.ts"; // A real-world thing (person, org, deal, ...) a document or chunk mentions. // Kept lightweight — identity keys only (email, domain, ...), not another @@ -12,11 +17,18 @@ export const MemoryEntitySchema = type({ }); export type MemoryEntity = typeof MemoryEntitySchema.infer; -export const MemoryEdgeRefTypeSchema = type("'document'|'entity'|'native'"); +// Adapter-facing endpoint kinds. `native` is a planning-time hint resolved +// to an entity row at the capture write boundary; it is never stored on +// memory.edge. +export const MemoryEdgeRefTypeSchema = type( + arktypeStringUnion(EDGE_REF_TYPES_ADAPTER) as + "'document'|'version'|'chunk'|'entity'|'native'", +); export type MemoryEdgeRefType = typeof MemoryEdgeRefTypeSchema.infer; export const MemoryEdgeRelSchema = type( - "'about'|'produced_by'|'links'|'parent'|'mentions'|'waiting_on'", + arktypeStringUnion(EDGE_RELS) as + "'mentions'|'about'|'authored_by'|'involves'|'part_of'|'derived_from'|'supports'|'contradicts'|'supersedes'", ); export type MemoryEdgeRel = typeof MemoryEdgeRelSchema.infer; diff --git a/src/core/schemas/search.test.ts b/src/core/schemas/search.test.ts index 0ce09fb..79f45d3 100644 --- a/src/core/schemas/search.test.ts +++ b/src/core/schemas/search.test.ts @@ -47,6 +47,28 @@ describe("SearchHitSchema", () => { const out = SearchHitSchema(fixture); expect(out instanceof type.errors).toBe(true); }); + + it("accepts additive attribution fields (CL-5870)", () => { + const fixture = fullHitFixture(); + fixture.provenance = "inferred"; + fixture.source_class = "derived"; + fixture.temporal_class = "lesson"; + fixture.occurred_at = "2026-07-01T00:00:00.000Z"; + fixture.valid_until = null; + fixture.supports = 3; + fixture.contradicts = 1; + fixture.derived_from = ["kv_src"]; + fixture.generator_agent_id = "resident-distiller"; + fixture.created_by_kind = "agent"; + const out = SearchHitSchema(fixture); + expect(out instanceof type.errors ? out.summary : out).toEqual(fixture); + }); + + it("still parses without attribution fields (additive / back-compat)", () => { + const fixture = fullHitFixture(); + const out = SearchHitSchema(fixture); + expect(out instanceof type.errors).toBe(false); + }); }); describe("SearchResponseSchema", () => { diff --git a/src/core/schemas/search.ts b/src/core/schemas/search.ts index 22c5c7e..dc19ee7 100644 --- a/src/core/schemas/search.ts +++ b/src/core/schemas/search.ts @@ -1,5 +1,11 @@ import { type } from "arktype"; -import { CreatedByKindSchema, MemoryVersionStatusSchema } from "./document.ts"; +import { + CreatedByKindSchema, + LineageClassSchema, + MemoryVersionStatusSchema, + ProvenanceModeSchema, + TemporalClassSchema, +} from "./document.ts"; // The retrieval contract locked on day one. A SearchHit always pins a // version_id (a citation must be reproducible against the exact version it @@ -41,6 +47,15 @@ export const SearchHitSchema = type({ citation: SearchHitCitationSchema, entity_ids: "string[]", channels_matched: SearchChannelSchema.array(), + // Additive attribution (CL-5870) — optional so older fixtures still parse. + "provenance?": ProvenanceModeSchema, + "source_class?": LineageClassSchema, + "temporal_class?": TemporalClassSchema, + "occurred_at?": "string", + "valid_until?": "string | null", + "supports?": "number.integer >= 0", + "contradicts?": "number.integer >= 0", + "derived_from?": "string[]", }); export type SearchHit = typeof SearchHitSchema.infer; diff --git a/src/db/schema.ts b/src/db/schema.ts index d69594e..2e25bba 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -1,4 +1,5 @@ import { + bigint, boolean, customType, integer, @@ -72,11 +73,23 @@ export const memoryVersion = memorySchema.table( actorCount: integer("actor_count").notNull().default(1), hasSocialSignal: boolean("has_social_signal").notNull().default(false), sourceClass: text("source_class").notNull().default("native"), + // How content was obtained: stated (asserted), inferred (derived claim), + // unknown (legacy / unset). Orthogonal to created_by_kind and source_class. + provenance: text("provenance").notNull().default("unknown"), + // Ranking temporal class + optional validity window (docs/TEMPORAL.md). + temporalClass: text("temporal_class").notNull().default("event"), + validFrom: timestamp("valid_from"), + validUntil: timestamp("valid_until"), + // Retention class (CL-5871) — when to forget, not how to rank. + retentionClass: text("retention_class").notNull().default("standard"), rawCaptureId: text("raw_capture_id").references(() => rawCapture.id), // Replay-generation tag — 'live' for the normal /capture path; a replay tags every // version it writes with its own transform_run id instead, so a replayed // corpus never collides with (or supersedes) the live one. generation: text("generation").notNull().default("live"), + // Monotonic commit marker for capture-feed pull (CL-5868). Assigned by + // Postgres bigserial (see migrations/0006_capture_feed.sql); not set in app. + feedSeq: bigint("feed_seq", { mode: "number" }), }, (t) => [ uniqueIndex("version_document_generation_version_uniq").on( @@ -85,6 +98,7 @@ export const memoryVersion = memorySchema.table( t.version, ), index("version_document_status_idx").on(t.documentId, t.status), + index("version_feed_seq_idx").on(t.tenantId, t.generation, t.feedSeq), ], ); @@ -227,6 +241,11 @@ export const transformRun = memorySchema.table( error: text("error"), createdAt: timestamp("created_at").notNull().defaultNow(), completedAt: timestamp("completed_at"), + /** Generation tag assigned to the prior live corpus on promote (for demote). */ + archivedLiveGeneration: text("archived_live_generation"), + /** Pre-promote active embed model_key (for demote dense restore). */ + archivedLiveModelKey: text("archived_live_model_key"), + promotedAt: timestamp("promoted_at"), }, (t) => [ uniqueIndex("transform_run_generation_uniq").on(t.generation), diff --git a/src/distiller/claim.test.ts b/src/distiller/claim.test.ts new file mode 100644 index 0000000..d4e18be --- /dev/null +++ b/src/distiller/claim.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "bun:test"; + +import { + buildDistilledClaim, + resolveNextCursor, + shouldProcessFeedEntry, +} from "./claim.ts"; +import { RESIDENT_DISTILLER_AGENT_ID } from "./constants.ts"; + +describe("buildDistilledClaim", () => { + it("sets inferred/derived identity and copies access tags", () => { + const claim = buildDistilledClaim({ + title: "Decision", + text: "Ship the feed first", + sourceAccessTags: ["memory.owner:u1", "memory.doc:d1"], + derivedFromVersionIds: ["kv_src"], + temporalClass: "lesson", + }); + expect(claim.generator_agent_id).toBe(RESIDENT_DISTILLER_AGENT_ID); + expect(claim.provenance).toBe("inferred"); + expect(claim.lineage_class).toBe("derived"); + expect(claim.derived_from).toEqual(["kv_src"]); + expect(claim.access_tags).toEqual(["memory.owner:u1", "memory.doc:d1"]); + expect(claim.temporal_class).toBe("lesson"); + }); +}); + +describe("shouldProcessFeedEntry", () => { + it("skips own generator writes", () => { + expect( + shouldProcessFeedEntry({ + versionId: "v1", + generatorAgentId: RESIDENT_DISTILLER_AGENT_ID, + }), + ).toBe(false); + }); + + it("accepts human / other agent writes", () => { + expect( + shouldProcessFeedEntry({ versionId: "v1", generatorAgentId: null }), + ).toBe(true); + expect( + shouldProcessFeedEntry({ + versionId: "v1", + generatorAgentId: "other-bot", + }), + ).toBe(true); + }); +}); + +describe("resolveNextCursor", () => { + it("returns page cursor even on poison (fail-soft)", () => { + expect(resolveNextCursor({ nextCursor: 42 }, { poison: true })).toBe(42); + expect(resolveNextCursor({ nextCursor: null })).toBe(null); + }); +}); diff --git a/src/distiller/claim.ts b/src/distiller/claim.ts new file mode 100644 index 0000000..38c148f --- /dev/null +++ b/src/distiller/claim.ts @@ -0,0 +1,89 @@ +import { RESIDENT_DISTILLER_AGENT_ID } from "./constants.ts"; + +/** + * Wire body fragment for a distilled claim write (memory_add / HTTP add). + * Access tags must be ≤ source entry tags — never widen. + */ +export type DistilledClaimWrite = { + title: string; + text: string; + access_tags: string[]; + generator_agent_id: string; + provenance: "inferred"; + lineage_class: "derived"; + derived_from: string[]; + kind?: string; + temporal_class?: "event" | "deadline" | "state" | "lesson"; + valid_from?: string; + valid_until?: string; +}; + +export type BuildDistilledClaimArgs = { + title: string; + text: string; + /** Source feed entry access tags (copied, not widened). */ + sourceAccessTags: readonly string[]; + /** Source version id(s) this claim is derived from. */ + derivedFromVersionIds: readonly string[]; + generatorAgentId?: string; + kind?: string; + temporalClass?: "event" | "deadline" | "state" | "lesson"; + validFrom?: string; + validUntil?: string; +}; + +/** Build a claim write body with loop-safe generator id and derived lineage. */ +export function buildDistilledClaim( + args: BuildDistilledClaimArgs, +): DistilledClaimWrite { + const claim: DistilledClaimWrite = { + title: args.title, + text: args.text, + access_tags: [...args.sourceAccessTags], + generator_agent_id: args.generatorAgentId ?? RESIDENT_DISTILLER_AGENT_ID, + provenance: "inferred", + lineage_class: "derived", + derived_from: [...args.derivedFromVersionIds], + }; + if (args.kind !== undefined) claim.kind = args.kind; + if (args.temporalClass !== undefined) { + claim.temporal_class = args.temporalClass; + } + if (args.validFrom !== undefined) claim.valid_from = args.validFrom; + if (args.validUntil !== undefined) claim.valid_until = args.validUntil; + return claim; +} + +export type FeedEntryLike = { + versionId: string; + generatorAgentId?: string | null; + kind?: string; + title?: string; + accessTags?: readonly string[]; +}; + +/** + * Gate: skip own writes (defense in depth — feed excludeGenerator is primary). + * Host policy can wrap this for kind/action-authority filters. + */ +export function shouldProcessFeedEntry( + entry: FeedEntryLike, + generatorAgentId: string = RESIDENT_DISTILLER_AGENT_ID, +): boolean { + if (entry.generatorAgentId === generatorAgentId) return false; + return true; +} + +/** + * Cursor advance: after processing a page, store nextCursor only when the + * host finished handling (including fail-soft poison quarantine). + * Pure helper — no I/O. + */ +export function resolveNextCursor( + page: { nextCursor: number | null }, + opts?: { poison?: boolean }, +): number | null { + // Fail-soft: still advance so a poison entry cannot block the feed forever. + if (opts?.poison) return page.nextCursor; + return page.nextCursor; +} diff --git a/src/distiller/constants.ts b/src/distiller/constants.ts new file mode 100644 index 0000000..ba9dea0 --- /dev/null +++ b/src/distiller/constants.ts @@ -0,0 +1,8 @@ +/** Stable generator id for the resident distiller — use with feed excludeGenerator. */ +export const RESIDENT_DISTILLER_AGENT_ID = "resident-distiller"; + +/** Default schedule: every 5 minutes. */ +export const RESIDENT_DISTILLER_CRON_DEFAULT = "*/5 * * * *"; + +/** Default workflow id when host does not override. */ +export const RESIDENT_DISTILLER_WORKFLOW_ID = "resident-memory-distiller"; diff --git a/src/distiller/index.ts b/src/distiller/index.ts new file mode 100644 index 0000000..4f63e78 --- /dev/null +++ b/src/distiller/index.ts @@ -0,0 +1,43 @@ +/** + * Resident distiller — first-class DX for Corbits apps. + * + * Two ways to run: + * + * 1. **Workflow** (recommended on Interchange): + * `createResidentDistiller({ inference })` → deploy the workflow. + * + * 2. **Imperative tick** (any scheduler): + * `runDistillTick({ client, distill, after })` with your model in `distill`. + * + * Substrate (feed, claim-bearing add, attribution, retention) lives in the + * memory plane; this module is the easy on-ramp, not a separate package. + */ +export { + RESIDENT_DISTILLER_AGENT_ID, + RESIDENT_DISTILLER_CRON_DEFAULT, + RESIDENT_DISTILLER_WORKFLOW_ID, +} from "./constants.ts"; + +export { + buildDistilledClaim, + resolveNextCursor, + shouldProcessFeedEntry, + type BuildDistilledClaimArgs, + type DistilledClaimWrite, + type FeedEntryLike, +} from "./claim.ts"; + +export { + runDistillTick, + type DistillOutcome, + type DistillTickFeedEntry, + type DistillTickPage, + type DistillTickResult, + type RunDistillTickArgs, +} from "./tick.ts"; + +export { + createResidentDistiller, + type CreateResidentDistillerOpts, + type ResidentDistiller, +} from "./workflow.ts"; diff --git a/src/distiller/tick.test.ts b/src/distiller/tick.test.ts new file mode 100644 index 0000000..c85383c --- /dev/null +++ b/src/distiller/tick.test.ts @@ -0,0 +1,142 @@ +import { describe, expect, it } from "bun:test"; + +import type { MemoryHttpClient } from "../tools/client.ts"; +import { RESIDENT_DISTILLER_AGENT_ID } from "./constants.ts"; +import { runDistillTick } from "./tick.ts"; + +function fakeClient(opts: { + page: unknown; + adds?: unknown[]; +}): MemoryHttpClient { + const adds = opts.adds ?? []; + return { + async add(body) { + adds.push(body); + return { documentId: "d_new", versionId: "v_new" }; + }, + async search() { + return { items: [] }; + }, + async list() { + return { events: [] }; + }, + async feed() { + return opts.page; + }, + }; +} + +describe("runDistillTick", () => { + it("writes claims and advances cursor", async () => { + const adds: unknown[] = []; + const client = fakeClient({ + adds, + page: { + entries: [ + { + feedSeq: 1, + versionId: "v1", + documentId: "d1", + kind: "note", + title: "Raw", + status: "active", + createdByKind: "human", + generatorAgentId: null, + provenance: "stated", + occurredAt: "2026-01-01T00:00:00.000Z", + createdAt: "2026-01-01T00:00:00.000Z", + accessTags: ["memory.owner:u1"], + }, + ], + nextCursor: 1, + }, + }); + + const result = await runDistillTick({ + client, + after: 0, + distill: async () => ({ + action: "write", + title: "Claim", + text: "Durable fact", + temporalClass: "state", + }), + }); + + expect(result.nextCursor).toBe(1); + expect(result.wrote).toBe(1); + expect(result.skipped).toBe(0); + expect(adds).toHaveLength(1); + const body = adds[0] as Record; + expect(body["generator_agent_id"]).toBe(RESIDENT_DISTILLER_AGENT_ID); + expect(body["derived_from"]).toEqual(["v1"]); + expect(body["access_tags"]).toEqual(["memory.owner:u1"]); + }); + + it("fail-soft poisons and still advances", async () => { + const client = fakeClient({ + page: { + entries: [ + { + feedSeq: 3, + versionId: "v3", + documentId: "d3", + kind: "note", + title: "Bad", + status: "active", + createdByKind: "human", + generatorAgentId: null, + provenance: "stated", + occurredAt: "2026-01-01T00:00:00.000Z", + createdAt: "2026-01-01T00:00:00.000Z", + accessTags: [], + }, + ], + nextCursor: 3, + }, + }); + + const result = await runDistillTick({ + client, + after: 2, + distill: async () => { + throw new Error("model blew up"); + }, + }); + + expect(result.poisoned).toBe(1); + expect(result.wrote).toBe(0); + expect(result.nextCursor).toBe(3); + }); + + it("skips when distill returns skip", async () => { + const client = fakeClient({ + page: { + entries: [ + { + feedSeq: 2, + versionId: "v2", + documentId: "d2", + kind: "note", + title: "Noise", + status: "active", + createdByKind: "human", + generatorAgentId: null, + provenance: "stated", + occurredAt: "2026-01-01T00:00:00.000Z", + createdAt: "2026-01-01T00:00:00.000Z", + accessTags: [], + }, + ], + nextCursor: 2, + }, + }); + + const result = await runDistillTick({ + client, + distill: async () => ({ action: "skip" }), + }); + expect(result.skipped).toBe(1); + expect(result.wrote).toBe(0); + }); +}); diff --git a/src/distiller/tick.ts b/src/distiller/tick.ts new file mode 100644 index 0000000..aca4c62 --- /dev/null +++ b/src/distiller/tick.ts @@ -0,0 +1,187 @@ +/** + * Imperative distill tick — for hosts that want a function, not a workflow. + * Inference stays injected: host supplies `distill` (call your model). + */ +import type { MemoryHttpClient } from "../tools/client.ts"; +import { + buildDistilledClaim, + shouldProcessFeedEntry, + type DistilledClaimWrite, +} from "./claim.ts"; +import { RESIDENT_DISTILLER_AGENT_ID } from "./constants.ts"; + +export type DistillTickFeedEntry = { + feedSeq: number; + versionId: string; + documentId: string; + kind: string; + title: string; + status: string; + createdByKind: string; + generatorAgentId: string | null; + provenance: string; + occurredAt: string; + createdAt: string; + accessTags: string[]; +}; + +export type DistillTickPage = { + entries: DistillTickFeedEntry[]; + nextCursor: number | null; +}; + +export type DistillOutcome = + | { action: "skip" } + | { action: "poison"; reason?: string } + | { + action: "write"; + title: string; + text: string; + kind?: string; + temporalClass?: "event" | "deadline" | "state" | "lesson"; + }; + +export type RunDistillTickArgs = { + client: MemoryHttpClient; + /** Exclusive cursor (last processed feed_seq). Default 0. */ + after?: number; + limit?: number; + generatorAgentId?: string; + /** + * Host inference: classify + distill one feed entry. + * Return skip / poison / write. Never throw for poison — use { action: "poison" }. + */ + distill: (entry: DistillTickFeedEntry) => Promise; + signal?: AbortSignal; +}; + +export type DistillTickResult = { + /** Cursor to persist for the next tick. */ + nextCursor: number; + processed: number; + wrote: number; + skipped: number; + poisoned: number; + claims: DistilledClaimWrite[]; +}; + +function parseFeedPage(raw: unknown): DistillTickPage { + if (raw === null || typeof raw !== "object") { + throw new Error("distill tick: feed response is not an object"); + } + const o = raw as Record; + const entriesRaw = o["entries"]; + if (!Array.isArray(entriesRaw)) { + throw new Error("distill tick: feed.entries missing"); + } + const entries: DistillTickFeedEntry[] = entriesRaw.map((e, i) => { + if (e === null || typeof e !== "object") { + throw new Error(`distill tick: feed.entries[${i}] invalid`); + } + const row = e as Record; + return { + feedSeq: Number(row["feedSeq"]), + versionId: String(row["versionId"] ?? ""), + documentId: String(row["documentId"] ?? ""), + kind: String(row["kind"] ?? "note"), + title: String(row["title"] ?? ""), + status: String(row["status"] ?? "active"), + createdByKind: String(row["createdByKind"] ?? "system"), + generatorAgentId: + row["generatorAgentId"] === null || row["generatorAgentId"] === undefined + ? null + : String(row["generatorAgentId"]), + provenance: String(row["provenance"] ?? "unknown"), + occurredAt: String(row["occurredAt"] ?? ""), + createdAt: String(row["createdAt"] ?? ""), + accessTags: Array.isArray(row["accessTags"]) + ? (row["accessTags"] as unknown[]).map(String) + : [], + }; + }); + const next = + o["nextCursor"] === null || o["nextCursor"] === undefined + ? null + : Number(o["nextCursor"]); + return { entries, nextCursor: next }; +} + +/** + * One distill tick: pull feed → host distill → write claims → return new cursor. + * Persist `nextCursor` after the tick succeeds (including poison advances). + */ +export async function runDistillTick( + args: RunDistillTickArgs, +): Promise { + const generatorAgentId = + args.generatorAgentId ?? RESIDENT_DISTILLER_AGENT_ID; + const after = args.after ?? 0; + + const raw = await args.client.feed( + { + after, + ...(args.limit !== undefined ? { limit: args.limit } : {}), + excludeGenerator: generatorAgentId, + }, + args.signal, + ); + const page = parseFeedPage(raw); + + let wrote = 0; + let skipped = 0; + let poisoned = 0; + const claims: DistilledClaimWrite[] = []; + + for (const entry of page.entries) { + if (!shouldProcessFeedEntry(entry, generatorAgentId)) { + skipped += 1; + continue; + } + let outcome: DistillOutcome; + try { + outcome = await args.distill(entry); + } catch (err) { + // Fail-soft: treat unexpected throw as poison so the cursor still advances. + poisoned += 1; + void err; + continue; + } + if (outcome.action === "skip") { + skipped += 1; + continue; + } + if (outcome.action === "poison") { + poisoned += 1; + continue; + } + + const claim = buildDistilledClaim({ + title: outcome.title, + text: outcome.text, + sourceAccessTags: entry.accessTags, + derivedFromVersionIds: [entry.versionId], + generatorAgentId, + ...(outcome.kind !== undefined ? { kind: outcome.kind } : {}), + ...(outcome.temporalClass !== undefined + ? { temporalClass: outcome.temporalClass } + : {}), + }); + await args.client.add(claim, args.signal); + claims.push(claim); + wrote += 1; + } + + const nextCursor = + page.nextCursor !== null && page.nextCursor !== undefined + ? page.nextCursor + : after; + + return { + nextCursor, + processed: page.entries.length, + wrote, + skipped, + poisoned, + claims, + }; +} diff --git a/src/distiller/workflow.test.ts b/src/distiller/workflow.test.ts new file mode 100644 index 0000000..f7f1956 --- /dev/null +++ b/src/distiller/workflow.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "bun:test"; + +import { + RESIDENT_DISTILLER_AGENT_ID, + RESIDENT_DISTILLER_WORKFLOW_ID, +} from "./constants.ts"; +import { createResidentDistiller } from "./workflow.ts"; + +describe("createResidentDistiller", () => { + it("returns a schedule workflow with memory tools on the agent", () => { + const { workflow, agent, generatorAgentId } = createResidentDistiller({ + inference: { + sources: [{ provider: "openai", model: "gpt-4.1-mini" }], + }, + }); + + expect(generatorAgentId).toBe(RESIDENT_DISTILLER_AGENT_ID); + expect(workflow.id).toBe(RESIDENT_DISTILLER_WORKFLOW_ID); + expect(workflow.triggers).toEqual([ + { type: "schedule", cron: "*/5 * * * *" }, + ]); + expect(agent.id).toBe(RESIDENT_DISTILLER_AGENT_ID); + expect(agent.toolFactories.length).toBeGreaterThanOrEqual(3); + expect(agent.systemPrompt).toContain("memory_feed"); + expect(agent.systemPrompt).toContain(RESIDENT_DISTILLER_AGENT_ID); + }); + + it("allows cron and id overrides", () => { + const { workflow, generatorAgentId, agent } = createResidentDistiller({ + id: "my-distiller", + agentId: "my-agent", + cron: "0 * * * *", + inference: { + sources: [{ provider: "openai", model: "gpt-4.1-mini" }], + }, + }); + expect(workflow.id).toBe("my-distiller"); + expect(generatorAgentId).toBe("my-agent"); + expect(workflow.triggers[0]).toEqual({ + type: "schedule", + cron: "0 * * * *", + }); + expect(agent.systemPrompt).toContain("my-agent"); + expect(agent.systemPrompt).not.toContain(RESIDENT_DISTILLER_AGENT_ID); + }); +}); diff --git a/src/distiller/workflow.ts b/src/distiller/workflow.ts new file mode 100644 index 0000000..b59656b --- /dev/null +++ b/src/distiller/workflow.ts @@ -0,0 +1,119 @@ +/** + * Ready-to-deploy resident distiller workflow for Interchange hosts. + * + * ```ts + * import { createResidentDistiller } from "@corbits/memory/distiller"; + * import { memoryAdd, memoryFeed, memorySearch } from "@corbits/memory/tools"; + * + * const workflow = createResidentDistiller({ + * inference: { sources: [{ provider: "openai", model: "gpt-4.1-mini" }] }, + * }); + * // deploy with host workflow-deploy + env: memoryBaseUrl, memoryTenantId, memoryAuthToken + * ``` + */ +import { + defineAgent, + type AgentDefinition, + type AnnotatedToolFactory, + type BaseEnv, + type InferencePreference, +} from "@intx/agent"; +import { defineWorkflow, type WorkflowDefinition } from "@intx/workflow"; + +import { memoryAdd } from "../tools/add.ts"; +import { memoryFeed } from "../tools/feed.ts"; +import { memorySearch } from "../tools/search.ts"; +import { MEMORY_CAPABILITY_IDS } from "../grant-requirements.ts"; +import { + RESIDENT_DISTILLER_AGENT_ID, + RESIDENT_DISTILLER_CRON_DEFAULT, + RESIDENT_DISTILLER_WORKFLOW_ID, +} from "./constants.ts"; + +const DEFAULT_SYSTEM_PROMPT = (generatorAgentId: string) => + `You are the resident memory distiller for a Corbits tenant. + +On each run: +1. Call memory_feed with after=, exclude_generator=${generatorAgentId}. +2. For each entry worth promoting to a durable claim: + - Classify (event / deadline / state / lesson) and gate junk / action-only noise. + - Write a concise claim via memory_add with: + generator_agent_id=${generatorAgentId} + provenance=inferred + lineage_class=derived + derived_from=[entry.versionId] + access_tags=entry.accessTags (copy exactly — never add broader tags) +3. Fail soft on poison entries: skip them and still advance past the page. +4. Remember the page nextCursor for the next run (host state / your notes). + +Never re-distill your own writes. Prefer few high-quality claims over many low-value ones. +When searching for corroboration, use memory_search and respect attribution (stated vs inferred).`; + +export type CreateResidentDistillerOpts = { + /** Workflow id (default resident-memory-distiller). */ + id?: string; + /** Agent id / generatorAgentId (default resident-distiller). */ + agentId?: string; + /** Cron schedule (default every 5 minutes). */ + cron?: string; + /** Host inference preferences (required for deploy hashing). */ + inference: { sources: readonly InferencePreference[] }; + /** Override system prompt. */ + systemPrompt?: string; + /** + * Extra tool factories beyond memory_feed / memory_add / memory_search. + * Default tools are always included first. + */ + extraTools?: readonly AnnotatedToolFactory[]; + /** Optional agent description. */ + description?: string; +}; + +export type ResidentDistiller = { + workflow: WorkflowDefinition; + agent: AgentDefinition; + generatorAgentId: string; +}; + +/** + * Build a schedule-triggered workflow + agent preloaded with memory tools. + * Host supplies inference sources and deploys with memory* env credentials. + */ +export function createResidentDistiller( + opts: CreateResidentDistillerOpts, +): ResidentDistiller { + const generatorAgentId = opts.agentId ?? RESIDENT_DISTILLER_AGENT_ID; + const tools = [ + memoryFeed, + memoryAdd, + memorySearch, + ...(opts.extraTools ?? []), + ] as AnnotatedToolFactory[]; + + const agent = defineAgent({ + id: generatorAgentId, + description: + opts.description ?? + "Resident memory distiller — feed → classify → claim write", + systemPrompt: opts.systemPrompt ?? DEFAULT_SYSTEM_PROMPT(generatorAgentId), + tools, + capabilities: [...MEMORY_CAPABILITY_IDS], + + inference: opts.inference, + tags: { + role: "resident-distiller", + package: "@corbits/memory", + }, + }); + + const workflow = defineWorkflow({ + id: opts.id ?? RESIDENT_DISTILLER_WORKFLOW_ID, + trigger: { + type: "schedule", + cron: opts.cron ?? RESIDENT_DISTILLER_CRON_DEFAULT, + }, + agent, + }); + + return { workflow, agent, generatorAgentId }; +} diff --git a/src/grant-requirements.test.ts b/src/grant-requirements.test.ts new file mode 100644 index 0000000..62aa248 --- /dev/null +++ b/src/grant-requirements.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { + MEMORY_CAPABILITY_IDS, + MEMORY_GRANT_REQUIREMENTS, +} from "./grant-requirements.ts"; + +describe("MEMORY_GRANT_REQUIREMENTS", () => { + test("covers add + search on memory resource", () => { + expect(MEMORY_GRANT_REQUIREMENTS.map((r) => r.action).sort()).toEqual([ + "add", + "search", + ]); + for (const r of MEMORY_GRANT_REQUIREMENTS) { + expect(r.resource).toBe("memory"); + expect(r.source).toBe("tenant"); + expect(r.surfaces).toContain("tools"); + expect(r.surfaces).toContain("distiller"); + } + }); + + test("capability ids are resource:action", () => { + expect([...MEMORY_CAPABILITY_IDS].sort()).toEqual([ + "memory:add", + "memory:search", + ]); + }); + + test("package.json interchange.grantRequirements stays in lockstep", () => { + const pkg = JSON.parse( + readFileSync(join(import.meta.dir, "..", "package.json"), "utf8"), + ) as { + interchange?: { + grantRequirements?: Array<{ + resource: string; + action: string; + source: string; + surfaces: string[]; + }>; + }; + }; + const fromPkg = pkg.interchange?.grantRequirements ?? []; + expect(fromPkg).toEqual( + MEMORY_GRANT_REQUIREMENTS.map((r) => ({ + resource: r.resource, + action: r.action, + source: r.source, + surfaces: [...r.surfaces], + })), + ); + }); +}); diff --git a/src/grant-requirements.ts b/src/grant-requirements.ts new file mode 100644 index 0000000..e52d530 --- /dev/null +++ b/src/grant-requirements.ts @@ -0,0 +1,49 @@ +/** + * Grant *requirements* for installers — not live grants. + * + * Mirrored under `package.json` → `interchange.grantRequirements` so a + * host installer can read npm metadata without executing code. The typed + * export is the in-repo SSOT; keep package.json in lockstep. + * + * Shape matches Interchange definition grant requirements + * (`resource` + `action` + `source`). Control plane materializes grants + * onto the workflow principal at deploy/launch. + */ + +export type MemoryGrantSource = "tenant" | "creator" | "invoker"; + +/** Package surfaces that need the requirement when installed. */ +export type MemoryGrantSurface = "tools" | "distiller" | "routes"; + +export type MemoryGrantRequirement = { + readonly resource: string; + readonly action: string; + /** Recommended authority source; installer/deploy may override. */ + readonly source: MemoryGrantSource; + readonly surfaces: readonly MemoryGrantSurface[]; +}; + +/** + * Minimum capability grants for memory tools / routes / process helpers. + * Document-tag access (`memory.doc:…`, `memory.space:…`) is separate and + * minted per document — not package install requirements. + */ +export const MEMORY_GRANT_REQUIREMENTS = [ + { + resource: "memory", + action: "add", + source: "tenant", + surfaces: ["tools", "distiller", "routes"], + }, + { + resource: "memory", + action: "search", + source: "tenant", + surfaces: ["tools", "distiller", "routes"], + }, +] as const satisfies readonly MemoryGrantRequirement[]; + +/** Compact `resource:action` form used on agent `capabilities` arrays. */ +export const MEMORY_CAPABILITY_IDS = MEMORY_GRANT_REQUIREMENTS.map( + (r) => `${r.resource}:${r.action}` as const, +); diff --git a/src/grant-tags.test.ts b/src/grant-tags.test.ts index d593655..5e2e4d9 100644 --- a/src/grant-tags.test.ts +++ b/src/grant-tags.test.ts @@ -43,13 +43,13 @@ describe("resolveAccessTags", () => { principalId: "u1", tenantId: "t1", accessTags: ["memory.space:eng"], - share: { tags: ["knowledge.project:ke"] }, + share: { tags: ["custom.project:ke"] }, }); expect(tags).toEqual( expect.arrayContaining([ ownerTag("u1"), "memory.space:eng", - "knowledge.project:ke", + "custom.project:ke", ]), ); }); diff --git a/src/grant-tags.ts b/src/grant-tags.ts index 3affe4b..36677fb 100644 --- a/src/grant-tags.ts +++ b/src/grant-tags.ts @@ -6,7 +6,8 @@ * - Capability checks (add/search on `memory`) live on the HTTP mount. * - Document access: creator always sees own docs; otherwise any `accessTag` * that `authorize(…, tag, "search")` allows. - * - Share sugars only mint tags — they never write grants. + * - Share sugars mint tags; peer grants are materialized separately when the + * host provides a WritableGrantStore (see services/share-grants.ts). */ import { authorize } from "@intx/authz"; import type { ConditionRegistry, GrantStore } from "@intx/authz"; diff --git a/src/http-bodies.ts b/src/http-bodies.ts index 716a957..d4ea242 100644 --- a/src/http-bodies.ts +++ b/src/http-bodies.ts @@ -25,6 +25,16 @@ export const AddRequest = type({ text: "string >= 1", "access_tags?": "string[]", "share?": ShareBody, + "kind?": "string", + /** Distiller / agent identity on the written version (loop-safety + attribution). */ + "generator_agent_id?": "string >= 1", + "provenance?": "'stated'|'inferred'|'unknown'", + "lineage_class?": "'native'|'imported'|'derived'", + "temporal_class?": "'event'|'deadline'|'state'|'lesson'", + /** Source version ids this claim is derived from (minted as derived_from edges). */ + "derived_from?": "string[]", + "valid_from?": "string", + "valid_until?": "string", }); export type AddRequest = typeof AddRequest.infer; @@ -36,6 +46,7 @@ export const SearchRequest = type({ "entity_ids?": "string[]", "sources?": "string[]", "includeEvidence?": "boolean", + "includeDeprecated?": "boolean", }); export type SearchRequest = typeof SearchRequest.infer; @@ -73,6 +84,48 @@ export function parseListLimitString( return n; } +/** HTTP query schema for GET /memory/feed. */ +export const FeedQuery = type({ + "after?": "string", + "limit?": "string", + "exclude_generator?": "string", +}); + +export type FeedQuery = typeof FeedQuery.infer; + +export type ParsedFeedQuery = { + after?: number; + limit?: number; + excludeGenerator?: string; +}; + +/** + * Parse feed query params. Returns `{ ok: false, error }` on invalid numbers. + */ +export function parseFeedQuery( + q: FeedQuery, +): { ok: true; value: ParsedFeedQuery } | { ok: false; error: string } { + const value: ParsedFeedQuery = {}; + if (q.after !== undefined && q.after !== "") { + const n = Number(q.after); + if (!Number.isInteger(n) || n < 0) { + return { ok: false, error: "after must be a non-negative integer" }; + } + value.after = n; + } + if (q.limit !== undefined && q.limit !== "") { + const n = Number(q.limit); + if (!Number.isInteger(n) || n < 1 || n > 100) { + return { ok: false, error: "limit must be an integer from 1 to 100" }; + } + value.limit = n; + } + if (q.exclude_generator !== undefined && q.exclude_generator !== "") { + value.excludeGenerator = q.exclude_generator; + } + return { ok: true, value }; +} + /** Coerce LLM-stringified integers before arktype number.integer checks. */ export function coerceOptionalLimitArg( args: Record, diff --git a/src/index.ts b/src/index.ts index 93eba33..4e16314 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,8 +6,10 @@ * `tenantId` in-process. Authz is the host grant store — this package * authenticates nothing itself. * - * Inference is host-owned and ephemeral (call your model, then add/search). - * Core does not mount an ingest agent or bake LLM into the write path. + * Distiller is first-class: `createResidentDistiller` / `runDistillTick` from + * `@corbits/memory/distiller` (or re-exported below). Inference stays host- + * injected; the package ships the workflow + tick helpers so apps opt in + * with a few lines. */ import type { Hono } from "hono"; import { createRequireGrant, type TenantEnv } from "@intx/hub-api"; @@ -40,9 +42,13 @@ export type { Memory, MemoryOptions, MemoryListParams, + MemoryFeedParams, + MemoryFeedEntry, + MemoryFeedResult, MemoryShare, SearchHit, SearchItem, + SearchAttribution, SearchResult, TextExtractor, TimelineEvent, @@ -56,10 +62,20 @@ export { LIST_LIMIT_MAX, } from "./memory.ts"; +// Installer discovery — grant *requirements* (not live grants) +export { + MEMORY_CAPABILITY_IDS, + MEMORY_GRANT_REQUIREMENTS, + type MemoryGrantRequirement, + type MemoryGrantSource, + type MemoryGrantSurface, +} from "./grant-requirements.ts"; + // Ports — pluggable storage and live sources export type { DocumentStore, DocumentStoreAddParams, + DocumentStoreAddResult, DocumentStoreSearchItem, DocumentStoreSearchParams, DocumentStoreSearchResult, @@ -74,6 +90,107 @@ export { createFakeSourceProvider, } from "./ports/fakes.ts"; +export type { WritableGrantStore } from "./ports/writable-grant-store.ts"; +export { + createInMemoryWritableGrantStore, + isWritableGrantStore, +} from "./ports/writable-grant-store.ts"; + +// Share materialization (CL-5873) +export { + buildShareGrants, + documentTag, + materializeShareGrants, + MEMORY_SHARE_CONDITION_KEY, + MEMORY_SHARE_CONDITION_REGISTRY, + shareWidenReceipt, + splitAudienceWiden, + type MaterializeShareGrantsInput, + type MemoryShareCondition, + type ShareWidenReceipt, +} from "./services/share-grants.ts"; + + +// Transform / replay surface (CL-5872) +export { + createTransformConfig, + demoteGeneration, + listTransformConfigs, + promoteGeneration, + resolveGenerationSearchParams, + runTransform, + TransformConfigNotFoundError, + TransformPromoteError, + type GenerationSearchParams, + type TransformConfigRow, + type TransformRunRow, +} from "./services/transform.ts"; + +// Capture feed (CL-5868) +export { + fetchFeed, + FEED_LIMIT_DEFAULT, + FEED_LIMIT_MAX, + FEED_LIMIT_MIN, + type FeedArgs, + type FeedEntry, + type FeedResult, +} from "./services/feed.ts"; + +// Retention / forgetting (CL-5871) +export { + deprecateVersion, + hardDeleteDocument, + setRetentionClass, + sweepEphemeral, + tombstoneDocument, + type RetentionMutationResult, +} from "./services/retention.ts"; + +// Resident distiller (CL-5869) — also `@corbits/memory/distiller` +export { + RESIDENT_DISTILLER_AGENT_ID, + RESIDENT_DISTILLER_CRON_DEFAULT, + RESIDENT_DISTILLER_WORKFLOW_ID, + buildDistilledClaim, + createResidentDistiller, + resolveNextCursor, + runDistillTick, + shouldProcessFeedEntry, + type BuildDistilledClaimArgs, + type CreateResidentDistillerOpts, + type DistillOutcome, + type DistillTickFeedEntry, + type DistillTickPage, + type DistillTickResult, + type DistilledClaimWrite, + type FeedEntryLike, + type ResidentDistiller, + type RunDistillTickArgs, +} from "./distiller/index.ts"; + +// Corroboration / living relevancy (CL-5867) +export { + corroborationFactor, + CORROBORATION_COUNT_LOG_CAP, + CORROBORATION_STRONG_FLOOR, + effectiveAuthority, + meetsStrongEvidenceGate, + type CorroborationCounts, + type StrongEvidenceSignals, +} from "./core/corroboration.ts"; + +// Embed model registry (ensure vs activate) +export { + activateEmbedModel, + activateEmbedModelByKey, + clearActiveEmbedModels, + ensureEmbedModel, + resolveActiveEmbedTable, + resolveEmbedTableByModelKey, +} from "./core/embed-model-registry.ts"; + + // Migrations export { runMemoryMigrations } from "./migrations.ts"; diff --git a/src/memory.test.ts b/src/memory.test.ts index 11fc1c4..562e0ad 100644 --- a/src/memory.test.ts +++ b/src/memory.test.ts @@ -514,7 +514,7 @@ async function freshPlane(opts?: { principalId: PRINCIPAL, content: { title: "T", text: "body" }, }); - expect(result).toEqual({ documentId: "kdoc_captured" }); + expect(result).toEqual({ documentId: "kdoc_captured", versionId: "kver_1" }); await plane.close(); }); @@ -534,7 +534,7 @@ async function freshPlane(opts?: { principalId: PRINCIPAL, content: { title: "T", text: "body" }, }); - expect(result).toEqual({ documentId: "kdoc_noop" }); + expect(result).toEqual({ documentId: "kdoc_noop", versionId: "kver_1" }); await plane.close(); }); diff --git a/src/memory.ts b/src/memory.ts index e1d2f9f..6a9d3f9 100644 --- a/src/memory.ts +++ b/src/memory.ts @@ -8,7 +8,7 @@ import { } from "./grant-tags.ts"; import type { EngineConfig } from "./config.ts"; -import { log } from "./log.ts"; +import { formatCaughtError, log } from "./log.ts"; import { createDb, type Db, type RawSql } from "./db/client.ts"; import { createFtsVerification, parseFtsLanguage } from "./core/fts-language.ts"; import { createRawSqlClient } from "./core/embed-sql.ts"; @@ -26,6 +26,38 @@ import { listTimelineEvents, type TimelineEvent, } from "./services/timeline.ts"; +import { + fetchFeed, + feedPageAfterAccessFilter, + type FeedEntry, +} from "./services/feed.ts"; + +import { + createTransformConfig, + demoteGeneration, + listTransformConfigs, + promoteGeneration, + runTransform, + type TransformConfigRow, + type TransformRunRow, +} from "./services/transform.ts"; +import { + deprecateVersion, + hardDeleteDocument, + setRetentionClass, + sweepEphemeral, + tombstoneDocument, +} from "./services/retention.ts"; +import { + documentTag, + materializeShareGrants, + MEMORY_SHARE_CONDITION_REGISTRY, +} from "./services/share-grants.ts"; + +import { + isWritableGrantStore, +} from "./ports/writable-grant-store.ts"; +import type { TransformConfigParams, TransformScope } from "./core/schemas/transform.ts"; import { LIVE_TIMEOUT_MS, mergeLocalLiveV1, @@ -115,6 +147,8 @@ export type MemorySearchParams = MemoryIdentity & { * Omit to include all mounted sources plus local. */ sources?: string[]; + /** Include deprecated versions in local retrieval (CL-5871). Default false. */ + includeDeprecated?: boolean; }; export type MemoryShare = ShareSugar; @@ -137,13 +171,51 @@ export type MemoryAddParams = MemoryIdentity & { */ accessTags?: string[]; /** - * Share sugar — only mints tags (tenant / peer owners / explicit tags). + * Share sugar — mints tags, and when the host grant store is writable, + * materializes peer grants on `memory.doc:` (CL-5873). */ share?: ShareSugar; attributes?: Record; + /** + * Claim / distiller identity (optional). When `generatorAgentId` is set the + * version is written as agent-authored so the capture feed can exclude it + * via `excludeGenerator` (loop-safety). + */ + generatorAgentId?: string; + provenance?: "stated" | "inferred" | "unknown"; + lineageClass?: "native" | "imported" | "derived"; + temporalClass?: "event" | "deadline" | "state" | "lesson"; + /** Source version ids → `derived_from` edges on the new version. */ + derivedFrom?: string[]; + validFrom?: string; + validUntil?: string; }; -export type MemoryAddResult = { documentId: string }; +export type MemoryAddResult = { + documentId: string; + versionId: string; + /** + * When `share.principals` was non-empty: true if peer grants were written + * to a WritableGrantStore; false if materialization was skipped (no writable + * store or soft failure). Omitted when no peer share was requested. + */ + grantsMaterialized?: boolean; +}; + +export type SearchAttribution = { + versionId: string; + provenance?: string; + sourceClass?: string; + temporalClass?: string; + createdByKind?: string; + generatorAgentId?: string | null; + occurredAt?: string; + validUntil?: string | null; + evidence?: "strong" | "weak" | "none"; + supports?: number; + contradicts?: number; + derivedFrom?: string[]; +}; export type SearchItem = { documentId: string; @@ -154,6 +226,8 @@ export type SearchItem = { citation: SearchHit["citation"]; /** ISO timestamp for merge recency when the store provides it. */ updatedAt?: string; + /** Additive provenance / temporal / corroboration for render-time attribution. */ + attribution?: SearchAttribution; }; export type SearchResult = { @@ -167,6 +241,34 @@ export type MemoryListParams = MemoryIdentity & { limit?: number; }; +export type MemoryFeedParams = MemoryIdentity & { + /** Exclusive cursor (last seen feedSeq). Default 0. */ + after?: number; + limit?: number; + excludeGenerator?: string; +}; + +export type MemoryFeedEntry = { + feedSeq: number; + versionId: string; + documentId: string; + kind: string; + title: string; + status: string; + createdByKind: string; + generatorAgentId: string | null; + provenance: string; + occurredAt: string; + createdAt: string; + /** Grant-pattern tags for claim writes (≤ source access). */ + accessTags: string[]; +}; + +export type MemoryFeedResult = { + entries: MemoryFeedEntry[]; + nextCursor: number | null; +}; + export class MemoryError extends Error { constructor( public readonly status: number, @@ -181,7 +283,62 @@ export type Memory = { search(params: MemorySearchParams): Promise; add(params: MemoryAddParams): Promise; list(params: MemoryListParams): Promise; + /** + * Cursor pull of new live versions (engine store only). Grant-checked like + * search. See docs/FEED.md. + */ + feed?(params: MemoryFeedParams): Promise; close(): Promise; + /** + * Transform / replay surface (engine DocumentStore only). Present when the + * plane was built with engine config; absent on custom/fake stores. + * Calling through a stub that omits these is a TypeScript error; runtime + * throws MemoryError(501) only if a partial implementation is forced. + */ + createTransformConfig?(input: { + tenantId: string; + name: string; + params: TransformConfigParams; + }): Promise; + listTransformConfigs?(tenantId: string): Promise; + runTransform?(input: { + configId: string; + scope?: TransformScope; + }): Promise; + promoteGeneration?(input: { + tenantId: string; + generation: string; + }): Promise; + demoteGeneration?(input: { + tenantId: string; + generation: string; + }): Promise; + /** + * Retention write paths (engine store only). See docs/RETENTION.md (CL-5871). + */ + deprecateVersion?(input: { + tenantId: string; + versionId: string; + reason?: string; + }): Promise<{ versionId: string; documentId: string; status: string } | null>; + tombstoneDocument?(input: { + tenantId: string; + documentId: string; + reason?: string; + }): Promise<{ versions: number }>; + hardDeleteDocument?(input: { + tenantId: string; + documentId: string; + }): Promise<{ deleted: boolean; reason?: string }>; + sweepEphemeral?(input: { + tenantId: string; + now?: Date; + }): Promise<{ versionsDeprecated: number }>; + setRetentionClass?(input: { + tenantId: string; + versionId: string; + retentionClass: "durable" | "standard" | "ephemeral" | "source_only"; + }): Promise<{ versionId: string; documentId: string; status: string } | null>; }; export type { TimelineEvent }; @@ -220,9 +377,14 @@ export function resolveGrantConfig( options: Pick, ): GrantConfig | undefined { if (!options.grantStore) return undefined; + // Merge memoryShare evaluator under host keys so share grants with + // conditions are not fail-closed-skipped by @intx/authz. return { grantStore: options.grantStore, - conditionRegistry: options.conditionRegistry ?? {}, + conditionRegistry: { + ...MEMORY_SHARE_CONDITION_REGISTRY, + ...(options.conditionRegistry ?? {}), + }, }; } @@ -258,7 +420,10 @@ function resolveListLimit(limit: number | undefined): number | undefined { return limit; } -function hitsToSearchItems(hits: readonly SearchHit[]): SearchItem[] { +function hitsToSearchItems( + hits: readonly SearchHit[], + evidence?: HybridSearchResult["evidence"], +): SearchItem[] { return hits.map((h) => ({ documentId: h.document_id, title: h.title, @@ -266,6 +431,24 @@ function hitsToSearchItems(hits: readonly SearchHit[]): SearchItem[] { score: h.score, kind: h.kind, citation: h.citation, + attribution: { + versionId: h.version_id, + createdByKind: h.created_by_kind, + ...(h.generator_agent_id !== undefined + ? { generatorAgentId: h.generator_agent_id } + : {}), + ...(h.provenance !== undefined ? { provenance: h.provenance } : {}), + ...(h.source_class !== undefined ? { sourceClass: h.source_class } : {}), + ...(h.temporal_class !== undefined + ? { temporalClass: h.temporal_class } + : {}), + ...(h.occurred_at !== undefined ? { occurredAt: h.occurred_at } : {}), + ...(h.valid_until !== undefined ? { validUntil: h.valid_until } : {}), + ...(h.supports !== undefined ? { supports: h.supports } : {}), + ...(h.contradicts !== undefined ? { contradicts: h.contradicts } : {}), + ...(h.derived_from !== undefined ? { derivedFrom: h.derived_from } : {}), + ...(evidence !== undefined ? { evidence } : {}), + }, })); } @@ -327,6 +510,8 @@ export function createMemory(options: MemoryOptions = {}): Memory { ...(grantStore !== undefined ? { grantStore } : {}), ...(conditionRegistry !== undefined ? { conditionRegistry } : {}), }); + + let transformDeps: EngineTransformDeps | undefined; const store = documentStore ?? (() => { @@ -336,8 +521,11 @@ export function createMemory(options: MemoryOptions = {}): Memory { "config is required when documentStore is not provided", ); } - return createEngineDocumentStore(config); + const engine = createEngineDocumentStore(config); + transformDeps = engine.deps; + return engine.store; })(); + return createPlaneFromStore( store, grants, @@ -345,6 +533,7 @@ export function createMemory(options: MemoryOptions = {}): Memory { ...(textExtractor ? { textExtractor } : {}), ...(sources ? { sources } : {}), }, + transformDeps, ); } @@ -452,14 +641,22 @@ function mergeToSearchResult(params: { ...(params.sources !== undefined ? { sources: params.sources } : {}), }); - const items: SearchItem[] = merged.items.map((it) => ({ - documentId: it.documentId, - title: it.title, - snippet: it.snippet, - score: it.score, - kind: it.kind, - citation: it.citation, - })); + const items: SearchItem[] = merged.items.map((it) => { + // Preserve local attribution when merge kept a local hit (same documentId). + const local = params.localItems.find((l) => l.documentId === it.documentId); + return { + documentId: it.documentId, + title: it.title, + snippet: it.snippet, + score: it.score, + kind: it.kind, + citation: it.citation, + ...(local?.attribution !== undefined + ? { attribution: local.attribution } + : {}), + ...(local?.updatedAt !== undefined ? { updatedAt: local.updatedAt } : {}), + }; + }); const degraded: DegradeFlag[] = [ ...(params.localDegraded ?? []), @@ -507,6 +704,7 @@ function createPlaneFromStore( store: DocumentStore, grants: GrantConfig | undefined, options: MemoryOptions, + transformDeps?: EngineTransformDeps, ): Memory { async function searchMerged( params: MemorySearchParams, @@ -527,6 +725,9 @@ function createPlaneFromStore( ...(params.entityIds !== undefined ? { entityIds: params.entityIds } : {}), + ...(params.includeDeprecated !== undefined + ? { includeDeprecated: params.includeDeprecated } + : {}), ...(grants !== undefined ? { grants: grants.grantStore } : {}), ...(grants?.conditionRegistry !== undefined ? { conditionRegistry: grants.conditionRegistry } @@ -540,6 +741,9 @@ function createPlaneFromStore( kind: it.kind, citation: it.citation, ...(it.updatedAt !== undefined ? { updatedAt: it.updatedAt } : {}), + ...(it.attribution !== undefined + ? { attribution: it.attribution } + : {}), })); localDegraded = local.degraded as DegradeFlag[] | undefined; localEvidence = local.evidence; @@ -628,7 +832,7 @@ function createPlaneFromStore( params.externalRef ?? `memory:${params.tenantId}:${crypto.randomUUID()}`; - return store.add({ + const result = await store.add({ tenantId: params.tenantId, principalId: params.principalId, title, @@ -640,7 +844,82 @@ function createPlaneFromStore( : {}), ...(params.adapter !== undefined ? { adapter: params.adapter } : {}), ...(params.kind !== undefined ? { kind: params.kind } : {}), + ...(params.generatorAgentId !== undefined + ? { generatorAgentId: params.generatorAgentId } + : {}), + ...(params.provenance !== undefined + ? { provenance: params.provenance } + : {}), + ...(params.lineageClass !== undefined + ? { lineageClass: params.lineageClass } + : {}), + ...(params.temporalClass !== undefined + ? { temporalClass: params.temporalClass } + : {}), + ...(params.derivedFrom !== undefined + ? { derivedFrom: params.derivedFrom } + : {}), + ...(params.validFrom !== undefined + ? { validFrom: params.validFrom } + : {}), + ...(params.validUntil !== undefined + ? { validUntil: params.validUntil } + : {}), }); + + // Share materialization (CL-5873): stamp document-scoped tag + write + // peer grants when the host store is writable. Tag mint alone is not + // enough for peers without host bootstrap grants on owner tags. The + // document is already durably committed by store.add() above, so a + // failure here must not throw (that would surface as add() failing + // for a document that in fact exists, inviting a caller retry that + // creates a duplicate) — it downgrades grantsMaterialized instead. + // grantsMaterialized is only ever true when BOTH the access tag was + // actually stamped AND the peer grant was actually written; either + // alone leaves canAccessDocument unable to find a match for peers. + let grantsMaterialized: boolean | undefined; + const peers = params.share?.principals; + if (peers && peers.length > 0) { + grantsMaterialized = false; + try { + const docTag = documentTag(result.documentId); + let tagStamped = false; + if (store.appendAccessTags) { + await store.appendAccessTags(params.tenantId, result.documentId, [docTag]); + tagStamped = true; + } else { + log.warn( + "memory.add: share.principals set but DocumentStore has no appendAccessTags; peer grants may not match", + { documentId: result.documentId }, + ); + } + if (isWritableGrantStore(grants?.grantStore)) { + await materializeShareGrants(grants.grantStore, { + tenantId: params.tenantId, + sharedByPrincipalId: params.principalId, + documentId: result.documentId, + sourceVersionId: result.versionId, + share: params.share ?? {}, + }); + grantsMaterialized = tagStamped; + } else { + log.warn( + "memory.add: share.principals set without WritableGrantStore; tags only (peers need host grants)", + { documentId: result.documentId }, + ); + } + } catch (err) { + log.error( + "memory.add: share materialization failed after document commit; peers may not have access", + { documentId: result.documentId, error: formatCaughtError(err) }, + ); + grantsMaterialized = false; + } + } + + return grantsMaterialized === undefined + ? result + : { ...result, grantsMaterialized }; }, async list(params) { @@ -656,20 +935,151 @@ function createPlaneFromStore( }); }, + async feed(params) { + if (!store.feed) { + throw new MemoryError( + 501, + "feed requires the engine DocumentStore", + ); + } + return store.feed({ + tenantId: params.tenantId, + principalId: params.principalId, + ...(params.after !== undefined ? { after: params.after } : {}), + ...(params.limit !== undefined ? { limit: params.limit } : {}), + ...(params.excludeGenerator !== undefined + ? { excludeGenerator: params.excludeGenerator } + : {}), + ...(grants !== undefined ? { grants: grants.grantStore } : {}), + ...(grants?.conditionRegistry !== undefined + ? { conditionRegistry: grants.conditionRegistry } + : {}), + }); + }, + async close() { await store.close(); }, + + async createTransformConfig(input) { + if (!transformDeps) { + throw new MemoryError( + 501, + "transform APIs require the engine DocumentStore", + ); + } + return createTransformConfig({ db: transformDeps.db }, input); + }, + + async listTransformConfigs(tenantId) { + if (!transformDeps) { + throw new MemoryError( + 501, + "transform APIs require the engine DocumentStore", + ); + } + return listTransformConfigs({ db: transformDeps.db }, tenantId); + }, + + async runTransform(input) { + if (!transformDeps) { + throw new MemoryError( + 501, + "transform APIs require the engine DocumentStore", + ); + } + return runTransform(transformDeps, input); + }, + + async promoteGeneration(input) { + if (!transformDeps) { + throw new MemoryError( + 501, + "transform APIs require the engine DocumentStore", + ); + } + return promoteGeneration(transformDeps, input); + }, + + async demoteGeneration(input) { + if (!transformDeps) { + throw new MemoryError( + 501, + "transform APIs require the engine DocumentStore", + ); + } + return demoteGeneration(transformDeps, input); + }, + + async deprecateVersion(input) { + if (!transformDeps) { + throw new MemoryError( + 501, + "retention APIs require the engine DocumentStore", + ); + } + return deprecateVersion(transformDeps.db, input); + }, + + async tombstoneDocument(input) { + if (!transformDeps) { + throw new MemoryError( + 501, + "retention APIs require the engine DocumentStore", + ); + } + return tombstoneDocument(transformDeps.db, input); + }, + + async hardDeleteDocument(input) { + if (!transformDeps) { + throw new MemoryError( + 501, + "retention APIs require the engine DocumentStore", + ); + } + return hardDeleteDocument(transformDeps.db, input); + }, + + async sweepEphemeral(input) { + if (!transformDeps) { + throw new MemoryError( + 501, + "retention APIs require the engine DocumentStore", + ); + } + return sweepEphemeral(transformDeps.db, input); + }, + + async setRetentionClass(input) { + if (!transformDeps) { + throw new MemoryError( + 501, + "retention APIs require the engine DocumentStore", + ); + } + return setRetentionClass(transformDeps.db, input); + }, }; return plane; } +type EngineTransformDeps = { + db: Db; + sql: RawSql; + config: EngineConfig; +}; + /** * Default DocumentStore: engine pgvector + hybrid search + timeline. * Owns construction-time rerank validation, FTS verification, and grant-tag * post-filter for document access. The plane never opens Postgres itself. */ -function createEngineDocumentStore(config: MemoryConfig): DocumentStore { +function createEngineDocumentStore(config: MemoryConfig): { + store: DocumentStore; + deps: EngineTransformDeps; +} { // Catch a chunk-size / reranker-limit mismatch at construction time, rather // than silently on every find once the reranker starts rejecting batches. // Throws instead of warning: a mismatch means every rerank call for this @@ -722,6 +1132,7 @@ function createEngineDocumentStore(config: MemoryConfig): DocumentStore { k?: number; kinds?: string[]; entityIds?: string[]; + includeDeprecated?: boolean; grants?: DocumentStoreSearchParams["grants"]; conditionRegistry?: DocumentStoreSearchParams["conditionRegistry"]; }): Promise { @@ -736,6 +1147,9 @@ function createEngineDocumentStore(config: MemoryConfig): DocumentStore { ...(params.entityIds !== undefined ? { entityIds: params.entityIds } : {}), + ...(params.includeDeprecated !== undefined + ? { includeDeprecated: params.includeDeprecated } + : {}), }); if (result.hits.length === 0) return result; @@ -812,80 +1226,196 @@ function createEngineDocumentStore(config: MemoryConfig): DocumentStore { } return { - async add(params) { - await ensureVerified(); + store: { + async add(params) { + await ensureVerified(); - const adapter = params.adapter ?? "http"; - const externalRef = - params.externalRef ?? - `memory:${params.tenantId}:${crypto.randomUUID()}`; - const accessTags = params.accessTags ?? [ownerTag(params.principalId)]; + const adapter = params.adapter ?? "http"; + const externalRef = + params.externalRef ?? + `memory:${params.tenantId}:${crypto.randomUUID()}`; + const accessTags = params.accessTags ?? [ownerTag(params.principalId)]; - const captureResult = await captureDocument(deps, { - tenantId: params.tenantId, - adapter, - occurredAt: new Date().toISOString(), - document: { - kind: params.kind ?? "note", - title: params.title, - externalRef, - accessTags, - entityHints: [], - chunks: [{ ordinal: 0, text: params.text }], - actor: { kind: "human", principalId: params.principalId }, - contentHash: "", // recomputed canonically in adapt-and-plan - ...(params.attributes !== undefined - ? { attributes: params.attributes } - : {}), - }, - }); - return { documentId: captureResult.documentId }; - }, + const generatorAgentId = params.generatorAgentId?.trim() || undefined; + const actor = generatorAgentId + ? { + kind: "agent" as const, + agentId: generatorAgentId, + principalId: params.principalId, + } + : { kind: "human" as const, principalId: params.principalId }; - async search(params) { - const result = await retrieve({ - tenantId: params.tenantId, - principalId: params.principalId, - query: params.query, - ...(params.limit !== undefined ? { k: params.limit } : {}), - ...(params.kinds !== undefined ? { kinds: params.kinds } : {}), - ...(params.entityIds !== undefined - ? { entityIds: params.entityIds } - : {}), - ...(params.grants !== undefined ? { grants: params.grants } : {}), - ...(params.conditionRegistry !== undefined - ? { conditionRegistry: params.conditionRegistry } - : {}), - }); - const items = hitsToSearchItems(result.hits); - if (params.includeEvidence) { + const edges = + params.derivedFrom && params.derivedFrom.length > 0 + ? params.derivedFrom.map((versionId) => ({ + rel: "derived_from" as const, + to: { type: "version" as const, ref: versionId }, + })) + : undefined; + + const captureResult = await captureDocument(deps, { + tenantId: params.tenantId, + adapter, + occurredAt: new Date().toISOString(), + document: { + kind: params.kind ?? "note", + title: params.title, + externalRef, + accessTags, + entityHints: [], + chunks: [{ ordinal: 0, text: params.text }], + actor, + contentHash: "", // recomputed canonically in adapt-and-plan + ...(params.attributes !== undefined + ? { attributes: params.attributes } + : {}), + ...(params.provenance !== undefined + ? { provenance: params.provenance } + : generatorAgentId + ? { provenance: "inferred" as const } + : {}), + ...(params.lineageClass !== undefined + ? { lineageClass: params.lineageClass } + : generatorAgentId + ? { lineageClass: "derived" as const } + : {}), + ...(params.temporalClass !== undefined + ? { temporalClass: params.temporalClass } + : {}), + ...(params.validFrom !== undefined + ? { validFrom: params.validFrom } + : {}), + ...(params.validUntil !== undefined + ? { validUntil: params.validUntil } + : {}), + ...(edges !== undefined ? { edges } : {}), + }, + }); + return { + documentId: captureResult.documentId, + versionId: captureResult.versionId, + }; + }, + + async appendAccessTags(tenantId, documentId, tags) { + if (tags.length === 0) return; + // Union into existing access_tags array (postgres text[]). Tenant + // filter is defense-in-depth against a forged documentId. + await sql` + UPDATE "memory"."document" + SET access_tags = ( + SELECT ARRAY( + SELECT DISTINCT t + FROM unnest( + COALESCE(access_tags, '{}'::text[]) || ${[...tags]}::text[] + ) AS t + ) + ) + WHERE id = ${documentId} + AND tenant_id = ${tenantId} + `; + }, + + async search(params) { + const result = await retrieve({ + tenantId: params.tenantId, + principalId: params.principalId, + query: params.query, + ...(params.limit !== undefined ? { k: params.limit } : {}), + ...(params.kinds !== undefined ? { kinds: params.kinds } : {}), + ...(params.entityIds !== undefined + ? { entityIds: params.entityIds } + : {}), + ...(params.includeDeprecated !== undefined + ? { includeDeprecated: params.includeDeprecated } + : {}), + ...(params.grants !== undefined ? { grants: params.grants } : {}), + ...(params.conditionRegistry !== undefined + ? { conditionRegistry: params.conditionRegistry } + : {}), + }); + const items = hitsToSearchItems(result.hits, result.evidence); + if (params.includeEvidence) { + return { + items, + evidence: result.evidence, + ...(result.degraded ? { degraded: result.degraded } : {}), + }; + } return { items, - evidence: result.evidence, ...(result.degraded ? { degraded: result.degraded } : {}), }; - } - return { - items, - ...(result.degraded ? { degraded: result.degraded } : {}), - }; - }, + }, - async list(params) { - return listTimelineEvents({ - db, - tenantId: params.tenantId, - principalId: params.principalId, - ...(params.limit !== undefined ? { limit: params.limit } : {}), - ...(params.grants !== undefined ? { grants: params.grants } : {}), - ...(params.conditionRegistry !== undefined - ? { conditionRegistry: params.conditionRegistry } - : {}), - }); - }, + async list(params) { + return listTimelineEvents({ + db, + tenantId: params.tenantId, + principalId: params.principalId, + ...(params.limit !== undefined ? { limit: params.limit } : {}), + ...(params.grants !== undefined ? { grants: params.grants } : {}), + ...(params.conditionRegistry !== undefined + ? { conditionRegistry: params.conditionRegistry } + : {}), + }); + }, - async close() { - await sql.end({ timeout: 5 }); + async feed(params) { + await ensureVerified(); + const raw = await fetchFeed(db, { + tenantId: params.tenantId, + ...(params.after !== undefined ? { after: params.after } : {}), + ...(params.limit !== undefined ? { limit: params.limit } : {}), + ...(params.excludeGenerator !== undefined + ? { excludeGenerator: params.excludeGenerator } + : {}), + }); + + const allowed: FeedEntry[] = []; + for (const entry of raw.entries) { + if (!params.grants) { + if (entry.createdByPrincipalId === params.principalId) { + allowed.push(entry); + } + continue; + } + const ok = await canAccessDocument({ + grants: params.grants, + tenantId: params.tenantId, + principalId: params.principalId, + createdByPrincipalId: entry.createdByPrincipalId, + accessTags: entry.accessTags, + ...(params.conditionRegistry !== undefined + ? { conditionRegistry: params.conditionRegistry } + : {}), + }); + if (ok) allowed.push(entry); + } + + const entries = allowed.map((e) => ({ + feedSeq: e.feedSeq, + versionId: e.versionId, + documentId: e.documentId, + kind: e.kind, + title: e.title, + status: e.status, + createdByKind: e.createdByKind, + generatorAgentId: e.generatorAgentId, + provenance: e.provenance, + occurredAt: e.occurredAt, + createdAt: e.createdAt, + accessTags: e.accessTags, + })); + // nextCursor must advance past the *raw* page even when ACL filters + // every entry — otherwise a denied page stalls the consumer forever. + return feedPageAfterAccessFilter(raw, entries); + }, + + async close() { + await sql.end({ timeout: 5 }); + }, }, + deps, }; } diff --git a/src/mount-config.test.ts b/src/mount-config.test.ts index 5335262..28971ca 100644 --- a/src/mount-config.test.ts +++ b/src/mount-config.test.ts @@ -54,6 +54,8 @@ describe("loadMemoryConfig — EMBED_TIMEOUT_MS / RERANK_TIMEOUT_MS", () => { it("rejects a non-positive-integer EMBED_TIMEOUT_MS", () => { process.env.EMBED_TIMEOUT_MS = "not-a-number"; - expect(() => loadMemoryConfig()).toThrow("EMBED_TIMEOUT_MS must be a positive integer"); + expect(() => loadMemoryConfig()).toThrow( + "EMBED_TIMEOUT_MS must be a positive integer", + ); }); }); diff --git a/src/mount-config.ts b/src/mount-config.ts index 350929d..df608f4 100644 --- a/src/mount-config.ts +++ b/src/mount-config.ts @@ -49,7 +49,11 @@ function optionalIntEnv(name: string): number | undefined { /** * Build a config from environment variables — a convenience for env-driven - * deploys. Hosts may also construct `MemoryConfig` programmatically. + * deploys. Hosts may also construct `MemoryConfig` programmatically (pass + * `memory.databaseUrl` directly — no env required). + * + * Database URL resolution for env-driven loads: DATABASE_URL — same Postgres + * as the host is fine; tables live under the `memory` schema, not public. */ export function loadMemoryConfig(): MemoryConfig { return { diff --git a/src/ports/fakes.ts b/src/ports/fakes.ts index 8d63414..bfe31c1 100644 --- a/src/ports/fakes.ts +++ b/src/ports/fakes.ts @@ -86,7 +86,17 @@ export function createFakeDocumentStore(): DocumentStore { row.externalRef = params.externalRef; } docs.push(row); - return { documentId }; + return { documentId, versionId: `fake_ver_${seq}` }; + }, + + async appendAccessTags(tenantId, documentId, tags) { + const row = docs.find( + (d) => d.documentId === documentId && d.tenantId === tenantId, + ); + if (!row) return; + const set = new Set(row.accessTags); + for (const t of tags) set.add(t); + row.accessTags = [...set]; }, async search( diff --git a/src/ports/types.ts b/src/ports/types.ts index 8c3f785..65e0f72 100644 --- a/src/ports/types.ts +++ b/src/ports/types.ts @@ -35,6 +35,14 @@ export type DocumentStoreAddParams = { adapter?: string; /** Document kind (default engine store uses `"note"`). */ kind?: string; + /** Claim / distiller fields — engine store only; vendors may ignore. */ + generatorAgentId?: string; + provenance?: "stated" | "inferred" | "unknown"; + lineageClass?: "native" | "imported" | "derived"; + temporalClass?: "event" | "deadline" | "state" | "lesson"; + derivedFrom?: string[]; + validFrom?: string; + validUntil?: string; }; export type DocumentStoreSearchParams = { @@ -47,6 +55,8 @@ export type DocumentStoreSearchParams = { kinds?: string[]; /** Narrow local retrieval by linked entity ids (unset/`[]` = no filter). */ entityIds?: string[]; + /** Include deprecated versions (CL-5871). */ + includeDeprecated?: boolean; /** * Host grant store for grant-tag document access (default engine + fakes). * Vendor stores may ignore (principal-bucket only). @@ -66,6 +76,24 @@ export type DocumentStoreSearchItem = { adapter?: string; externalRef?: string; updatedAt?: string; + /** + * Optional attribution block (CL-5870). Engine store always fills this; + * vendor stores may omit. + */ + attribution?: { + versionId: string; + provenance?: string; + sourceClass?: string; + temporalClass?: string; + createdByKind?: string; + generatorAgentId?: string | null; + occurredAt?: string; + validUntil?: string | null; + evidence?: "strong" | "weak" | "none"; + supports?: number; + contradicts?: number; + derivedFrom?: string[]; + }; }; export type DocumentStoreSearchResult = { @@ -90,17 +118,70 @@ export type DocumentStoreListEvent = { principalId: string; }; +export type DocumentStoreFeedParams = { + tenantId: string; + principalId: string; + after?: number; + limit?: number; + excludeGenerator?: string; + grants?: GrantStore; + conditionRegistry?: ConditionRegistry; +}; + +export type DocumentStoreFeedEntry = { + feedSeq: number; + versionId: string; + documentId: string; + kind: string; + title: string; + status: string; + createdByKind: string; + generatorAgentId: string | null; + provenance: string; + occurredAt: string; + createdAt: string; + /** Grant-pattern tags — distiller copies onto claims (never widen). */ + accessTags: string[]; +}; + +export type DocumentStoreFeedResult = { + entries: DocumentStoreFeedEntry[]; + nextCursor: number | null; +}; + /** * Durable document plane. Default implementation is the engine's pgvector * store. Hosts inject a DocumentStore (or fakes) via `options.documentStore` * to replace local Postgres entirely — this is the only product path for * swapping backends. */ +export type DocumentStoreAddResult = { + documentId: string; + /** Active version written (or existing active on noop). */ + versionId: string; +}; + export type DocumentStore = { - add(params: DocumentStoreAddParams): Promise<{ documentId: string }>; + add(params: DocumentStoreAddParams): Promise; search(params: DocumentStoreSearchParams): Promise; list(params: DocumentStoreListParams): Promise; + /** + * Cursor pull of new live versions (CL-5868). Engine-only; vendor stores + * may omit (plane returns 501). + */ + feed?(params: DocumentStoreFeedParams): Promise; close(): Promise; + /** + * Append access tags after insert (used by share materialization to stamp + * `memory.doc:` once the document id is known). Optional — stores that + * omit it leave peer share grants without a matching tag (fail-closed). + * Tenant is required so a forged documentId cannot retag another tenant. + */ + appendAccessTags?( + tenantId: string, + documentId: string, + tags: readonly string[], + ): Promise; }; /** diff --git a/src/ports/writable-grant-store.ts b/src/ports/writable-grant-store.ts new file mode 100644 index 0000000..5d572bb --- /dev/null +++ b/src/ports/writable-grant-store.ts @@ -0,0 +1,51 @@ +/** + * Host grant store that can materialize share grants. + * + * `@intx/authz` `GrantStore` is read-only (`collectGrants`). Memory never + * owns the grant plane — hosts that want share-on-add to work pass a store + * implementing this write seam. + */ +import type { GrantRule, GrantStore } from "@intx/authz"; + +export type WritableGrantStore = GrantStore & { + /** + * Insert or replace a grant by id. Hosts map this onto their control-plane + * grant table. Memory only calls this for share materialization. + */ + putGrant(grant: GrantRule): Promise; +}; + +export function isWritableGrantStore( + store: GrantStore | undefined, +): store is WritableGrantStore { + return ( + store !== undefined && + typeof (store as WritableGrantStore).putGrant === "function" + ); +} + +/** + * In-memory writable store for tests. Collects by principalId like + * `createInMemoryGrantStore` (tenantId accepted, unused). + */ +export function createInMemoryWritableGrantStore( + initial: GrantRule[] = [], +): WritableGrantStore & { grants: GrantRule[] } { + const grants = [...initial]; + return { + grants, + async collectGrants(principalId: string, _tenantId?: string) { + const now = new Date(); + return grants.filter((g) => { + if (g.principalId !== principalId) return false; + if (g.expiresAt !== null && g.expiresAt <= now) return false; + return true; + }); + }, + async putGrant(grant: GrantRule) { + const idx = grants.findIndex((g) => g.id === grant.id); + if (idx >= 0) grants[idx] = grant; + else grants.push(grant); + }, + }; +} diff --git a/src/routes/add.ts b/src/routes/add.ts index 7e5a979..f484175 100644 --- a/src/routes/add.ts +++ b/src/routes/add.ts @@ -13,6 +13,7 @@ import { caller, grantGuard, requirePrincipal } from "./deps.ts"; const AddResponse = type({ documentId: "string", + versionId: "string", }); export function mountAddRoute(app: Hono, deps: RouteDeps): void { @@ -57,14 +58,36 @@ export function mountAddRoute(app: Hono, deps: RouteDeps): void { } try { - const { documentId } = await deps.memory.add({ + const result = await deps.memory.add({ content: { title, text }, tenantId: scopeId, principalId: subjectId, ...(accessTags !== undefined ? { accessTags } : {}), ...(share !== undefined ? { share } : {}), + ...(body.kind !== undefined ? { kind: body.kind } : {}), + ...(body.generator_agent_id !== undefined + ? { generatorAgentId: body.generator_agent_id } + : {}), + ...(body.provenance !== undefined + ? { provenance: body.provenance } + : {}), + ...(body.lineage_class !== undefined + ? { lineageClass: body.lineage_class } + : {}), + ...(body.temporal_class !== undefined + ? { temporalClass: body.temporal_class } + : {}), + ...(body.derived_from !== undefined + ? { derivedFrom: body.derived_from } + : {}), + ...(body.valid_from !== undefined + ? { validFrom: body.valid_from } + : {}), + ...(body.valid_until !== undefined + ? { validUntil: body.valid_until } + : {}), }); - return c.json({ documentId }); + return c.json({ documentId: result.documentId, versionId: result.versionId }); } catch (err) { if (err instanceof MemoryError) { return c.json( diff --git a/src/routes/feed.ts b/src/routes/feed.ts new file mode 100644 index 0000000..3f64abd --- /dev/null +++ b/src/routes/feed.ts @@ -0,0 +1,85 @@ +import type { Hono } from "hono"; +import type { TenantEnv } from "@intx/hub-api"; +import { describeRoute, resolver, validator } from "hono-openapi"; +import { type } from "arktype"; + +import { formatCaughtError, log } from "../log.ts"; +import { FeedQuery, parseFeedQuery } from "../http-bodies.ts"; +import { MemoryError } from "../memory.ts"; +import type { RouteDeps } from "./deps.ts"; +import { caller, grantGuard, requirePrincipal } from "./deps.ts"; + +const FeedResponse = type({ + entries: type({ + feedSeq: "number", + versionId: "string", + documentId: "string", + kind: "string", + title: "string", + status: "string", + createdByKind: "string", + generatorAgentId: "string|null", + provenance: "string", + occurredAt: "string", + createdAt: "string", + accessTags: "string[]", + }).array(), + nextCursor: "number|null", +}); + +export function mountFeedRoute(app: Hono, deps: RouteDeps): void { + app.get( + "/api/tenants/:tenantId/memory/feed", + + describeRoute({ + tags: ["memory"], + summary: "Pull new live versions after a cursor (capture feed)", + responses: { + 200: { + description: "Ordered feed page", + content: { + "application/json": { schema: resolver(FeedResponse) }, + }, + }, + 400: { description: "Invalid query params" }, + 401: { description: "No principal on the request context" }, + 403: { description: "Missing the memory:search grant" }, + 501: { description: "Feed requires the engine DocumentStore" }, + 502: { description: "Feed query failed" }, + }, + }), + requirePrincipal(), + grantGuard(deps, "search"), + validator("query", FeedQuery), + async (c) => { + const { scopeId, subjectId } = caller(c); + const parsed = parseFeedQuery(c.req.valid("query")); + if (!parsed.ok) { + return c.json({ error: parsed.error }, 400); + } + if (!deps.memory.feed) { + return c.json({ error: "feed requires the engine DocumentStore" }, 501); + } + try { + const result = await deps.memory.feed({ + tenantId: scopeId, + principalId: subjectId, + ...parsed.value, + }); + return c.json(result); + } catch (err) { + if (err instanceof MemoryError) { + return c.json( + { error: err.message }, + err.status as 400 | 501, + ); + } + const errMessage = formatCaughtError(err); + log.error(`memory feed failed: ${errMessage}`, { + error: errMessage, + }); + return c.json({ error: "feed failed" }, 502); + } + }, + ); +} diff --git a/src/routes/mount.ts b/src/routes/mount.ts index 34d38ae..bc1a760 100644 --- a/src/routes/mount.ts +++ b/src/routes/mount.ts @@ -10,10 +10,11 @@ import type { RouteDeps } from "./deps.ts"; import { mountAddRoute } from "./add.ts"; import { mountSearchRoute } from "./search.ts"; import { mountListRoute } from "./list.ts"; +import { mountFeedRoute } from "./feed.ts"; export type { GrantConfig, RouteDeps } from "./deps.ts"; -/** HTTP JSON routes: add, search, list. */ +/** HTTP JSON routes: add, search, list, feed. */ export function registerMemoryRoutes( app: Hono, deps: RouteDeps, @@ -21,4 +22,5 @@ export function registerMemoryRoutes( mountAddRoute(app, deps); mountSearchRoute(app, deps); mountListRoute(app, deps); + mountFeedRoute(app, deps); } diff --git a/src/routes/routes.test.ts b/src/routes/routes.test.ts index 4607286..70bd434 100644 --- a/src/routes/routes.test.ts +++ b/src/routes/routes.test.ts @@ -50,7 +50,7 @@ function stubPlane(opts?: { tenantId: p.tenantId, principalId: p.principalId, }); - return { documentId: "doc-stub" }; + return { documentId: "doc-stub", versionId: "ver-stub" }; }, list: async (p) => { return catalog @@ -168,8 +168,9 @@ describe("memory HTTP routes", () => { jsonPost({ title: "t", text: "body" }), ); expect(res.status).toBe(200); - const body = (await res.json()) as { documentId: string }; + const body = (await res.json()) as { documentId: string; versionId: string }; expect(body.documentId).toBe("doc-stub"); + expect(body.versionId).toBe("ver-stub"); expect(added).toEqual([ { title: "t", tenantId: TENANT, principalId: PRINCIPAL }, ]); diff --git a/src/routes/search.ts b/src/routes/search.ts index 2dc0481..68edab0 100644 --- a/src/routes/search.ts +++ b/src/routes/search.ts @@ -25,6 +25,8 @@ const SearchResponse = type({ score: "number", kind: "string", citation: "unknown", + "attribution?": "unknown", + "updatedAt?": "string", }).array(), "evidence?": "'strong'|'weak'|'none'", "degraded?": "string[]", @@ -59,8 +61,15 @@ export function mountSearchRoute(app: Hono, deps: RouteDeps): void { grantGuard(deps, "search"), validator("json", SearchRequest), async (c) => { - const { query, limit, kinds, entity_ids, sources, includeEvidence } = - c.req.valid("json"); + const { + query, + limit, + kinds, + entity_ids, + sources, + includeEvidence, + includeDeprecated, + } = c.req.valid("json"); const { scopeId, subjectId } = caller(c); try { const result = await deps.memory.search({ @@ -74,6 +83,9 @@ export function mountSearchRoute(app: Hono, deps: RouteDeps): void { ...(kinds !== undefined ? { kinds } : {}), ...(entity_ids !== undefined ? { entityIds: entity_ids } : {}), ...(sources !== undefined ? { sources } : {}), + ...(includeDeprecated !== undefined + ? { includeDeprecated } + : {}), }); return c.json(result); } catch (err) { diff --git a/src/services/capture.ts b/src/services/capture.ts index 19a2fb9..6fcecc7 100644 --- a/src/services/capture.ts +++ b/src/services/capture.ts @@ -26,7 +26,7 @@ import type { } from "../core/schemas/adapted-document.ts"; import type { MemoryEdgeHint } from "../core/schemas/entity-edge.ts"; import { createRawSqlClient } from "../core/embed-sql.ts"; -import { activateEmbedModel } from "../core/embed-model-registry.ts"; +import { activateEmbedModel, ensureEmbedModel } from "../core/embed-model-registry.ts"; import type { EmbedClientConfig } from "../core/embed-client.ts"; import { embedChunks, type EmbeddableChunk } from "../core/embed-worker.ts"; import { toEmbedClientConfig } from "../core/engine-client-config.ts"; @@ -71,6 +71,10 @@ type CaptureTxResult = // defaults here rather than at the schema layer, so every capture (including // a future caller that forgets to set a signal) always produces a // well-formed AuthoritySignals rather than an undefined-riddled one. +// +// Ranking sourceClass (thread/channel/…) is deliberately NOT written to the +// version.source_class column — that column is data lineage +// (native|imported|derived) via lineageClass. See deriveLineageClass. function deriveAuthoritySignals(plan: CapturePlan): AuthoritySignals { return { createdByKind: plan.document.actor?.kind ?? "system", @@ -80,6 +84,26 @@ function deriveAuthoritySignals(plan: CapturePlan): AuthoritySignals { }; } +function deriveLineageClass(plan: CapturePlan): string { + return plan.document.lineageClass ?? "native"; +} + +function deriveProvenance(plan: CapturePlan): string { + return plan.document.provenance ?? "stated"; +} + +function deriveTemporalClass(plan: CapturePlan): string { + if (plan.document.temporalClass) return plan.document.temporalClass; + // Distilled claims default to state ranking; raw captures to event. + if ((plan.document.provenance ?? "stated") === "inferred") return "state"; + return "event"; +} + +function parseOptionalDate(value: string | undefined): Date | null { + if (!value) return null; + return new Date(value); +} + async function insertVersion( tx: Tx, input: CaptureInput, @@ -111,7 +135,11 @@ async function insertVersion( authority: computeAuthority(authoritySignals), actorCount: authoritySignals.actorCount, hasSocialSignal: authoritySignals.hasSocialSignal, - sourceClass: authoritySignals.sourceClass, + sourceClass: deriveLineageClass(plan), + provenance: deriveProvenance(plan), + temporalClass: deriveTemporalClass(plan), + validFrom: parseOptionalDate(plan.document.validFrom), + validUntil: parseOptionalDate(plan.document.validUntil), rawCaptureId: opts.rawCaptureId, generation: opts.generation, }); @@ -195,13 +223,14 @@ async function insertOrReuseRawCapture( // No unique constraint backs memory_entity — dedupe here on an exact // (tenantId, kind, identifiers) match, matching what a caller re-emits for -// the same real-world thing across captures. +// the same real-world thing across captures. Returns the entity id (existing +// or freshly inserted) so edge resolution can point at it. async function upsertEntity( tx: Tx, tenantId: string, hint: EntityHint, now: Date, -): Promise { +): Promise { const identifiers = { value: hint.identifier }; const rows = await tx .select({ @@ -215,23 +244,29 @@ async function upsertEntity( eq(memoryEntity.kind, hint.kind), ), ); - const exists = rows.some( + const match = rows.find( (r) => JSON.stringify(r.identifiers) === JSON.stringify(identifiers), ); - if (exists) return; + if (match) return match.id; + const id = newId("kent"); await tx.insert(memoryEntity).values({ - id: newId("kent"), + id, tenantId, kind: hint.kind, identifiers, createdAt: now, updatedAt: now, }); + return id; } // No unique constraint backs memory_edge either — dedupe on the full // (tenantId, rel, from, to) tuple so re-ingesting the same document doesn't // pile up duplicate relationship rows across versions. +// +// Adapter-facing `native` endpoints are planning-time hints for principals +// (or other non-entity refs). Resolve them to a memory_entity row before +// insert so the DB CHECK (document|version|chunk|entity) is always satisfied. async function upsertEdge( tx: Tx, tenantId: string, @@ -239,6 +274,18 @@ async function upsertEdge( hint: MemoryEdgeHint, now: Date, ): Promise { + let toType = hint.to.type; + let toRef = hint.to.ref; + if (toType === "native") { + toRef = await upsertEntity( + tx, + tenantId, + { kind: "principal", identifier: toRef }, + now, + ); + toType = "entity"; + } + const rows = await tx .select({ id: memoryEdge.id }) .from(memoryEdge) @@ -248,8 +295,8 @@ async function upsertEdge( eq(memoryEdge.rel, hint.rel), eq(memoryEdge.fromType, "document"), eq(memoryEdge.fromRef, documentId), - eq(memoryEdge.toType, hint.to.type), - eq(memoryEdge.toRef, hint.to.ref), + eq(memoryEdge.toType, toType), + eq(memoryEdge.toRef, toRef), ), ) .limit(1); @@ -260,8 +307,8 @@ async function upsertEdge( rel: hint.rel, fromType: "document", fromRef: documentId, - toType: hint.to.type, - toRef: hint.to.ref, + toType, + toRef, createdAt: now, }); } @@ -469,35 +516,38 @@ export { toEmbedClientConfig }; // Embeds a version's freshly-inserted chunks and stores their vectors, after // the derivation transaction has already committed. Best-effort in the -// fullest sense: ANY failure here — including activateEmbedModel's dims-probe -// network call, not just embedChunks' own client-error/rejected-chunk cases — -// is caught, logged, and swallowed. The chunk rows are already durable, and a -// later re-embed pass can pick up anything left unembedded (mirrors -// embed-worker.ts's pending-chunk contract, just invoked eagerly here instead -// of by polling). Returns whether embedding degraded so the caller can surface -// it. Shared by the live /capture path and a replay — the only difference -// between them is which `EmbedClientConfig` is passed in. +// fullest sense: ANY failure here — including ensure/activateEmbedModel's +// dims-probe network call, not just embedChunks' own client-error/rejected- +// chunk cases — is caught, logged, and swallowed. The chunk rows are already +// durable, and a later re-embed pass can pick up anything left unembedded +// (mirrors embed-worker.ts's pending-chunk contract, just invoked eagerly +// here instead of by polling). Returns whether embedding degraded so the +// caller can surface it. +// +// `promoteActive` (default true): live capture activates the model so dense +// search targets it. Replay must pass false so ensureEmbedModel only creates +// the table without flipping the tenant's active embed model (CL-5872). async function embedInsertedChunksWithConfig( sql: RawSql, tenantId: string, chunks: EmbeddableChunk[], embedClientConfig: EmbedClientConfig, + opts: { promoteActive?: boolean } = {}, ): Promise<{ degraded: boolean }> { if (chunks.length === 0) return { degraded: false }; try { const client = createRawSqlClient(sql); + const promoteActive = opts.promoteActive !== false; - const activeTable = await activateEmbedModel( - client, - tenantId, - embedClientConfig, - ); + const table = promoteActive + ? await activateEmbedModel(client, tenantId, embedClientConfig) + : await ensureEmbedModel(client, tenantId, embedClientConfig); const result = await embedChunks( client, tenantId, - activeTable, + table, chunks, embedClientConfig, ); @@ -599,6 +649,8 @@ export async function deriveFromRawCapture( input.tenantId, txResult.insertedChunks, derivation.embed, + // Replay never flips the tenant's active embed model. + { promoteActive: false }, ); return { diff --git a/src/services/feed.test.ts b/src/services/feed.test.ts new file mode 100644 index 0000000..c0b2c4e --- /dev/null +++ b/src/services/feed.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "bun:test"; +import { + FEED_LIMIT_DEFAULT, + FEED_LIMIT_MAX, + FEED_LIMIT_MIN, + FeedInputError, + feedPageAfterAccessFilter, +} from "./feed.ts"; + +describe("feed constants", () => { + it("bounds page size", () => { + expect(FEED_LIMIT_MIN).toBe(1); + expect(FEED_LIMIT_DEFAULT).toBeLessThanOrEqual(FEED_LIMIT_MAX); + expect(FEED_LIMIT_MAX).toBe(100); + }); +}); + +describe("FeedInputError", () => { + it("is a 400-class error", () => { + const err = new FeedInputError("bad cursor"); + expect(err.status).toBe(400); + expect(err.message).toBe("bad cursor"); + }); +}); + +describe("feedPageAfterAccessFilter", () => { + it("keeps raw nextCursor when ACL denies the whole page", () => { + const raw = { + entries: [{ feedSeq: 10 }, { feedSeq: 11 }], + nextCursor: 11, + }; + const page = feedPageAfterAccessFilter(raw, []); + expect(page.entries).toEqual([]); + expect(page.nextCursor).toBe(11); + }); + + it("keeps raw nextCursor when some entries are allowed", () => { + const raw = { + entries: [{ feedSeq: 1 }, { feedSeq: 2 }, { feedSeq: 3 }], + nextCursor: 3, + }; + const allowed = [{ feedSeq: 1 }]; + const page = feedPageAfterAccessFilter(raw, allowed); + expect(page.entries).toEqual(allowed); + // Not the last allowed feedSeq (1) — advance past examined raw page. + expect(page.nextCursor).toBe(3); + }); + + it("returns null nextCursor when raw page is empty (end of feed)", () => { + const page = feedPageAfterAccessFilter( + { entries: [], nextCursor: null }, + [], + ); + expect(page.nextCursor).toBeNull(); + }); +}); diff --git a/src/services/feed.ts b/src/services/feed.ts new file mode 100644 index 0000000..f46ff2c --- /dev/null +++ b/src/services/feed.ts @@ -0,0 +1,149 @@ +/** + * Capture feed — cursor pull of new versions for the resident distiller (CL-5868). + * See docs/FEED.md. + */ +import { and, asc, eq, gt, isNull, ne, or, sql } from "drizzle-orm"; + +import type { Db } from "../db/client.ts"; +import { + memoryDocument, + memoryVersion, +} from "../db/schema.ts"; +import { LIVE_GENERATION } from "../core/generation.ts"; + +export const FEED_LIMIT_MIN = 1; +export const FEED_LIMIT_MAX = 100; +export const FEED_LIMIT_DEFAULT = 50; + +export type FeedArgs = { + tenantId: string; + /** Exclusive cursor: return rows with feed_seq > after. Default 0. */ + after?: number; + limit?: number; + /** Skip versions written by this generator (loop-safe for distiller). */ + excludeGenerator?: string; +}; + +export type FeedEntry = { + feedSeq: number; + versionId: string; + documentId: string; + kind: string; + title: string; + status: string; + createdByKind: string; + generatorAgentId: string | null; + provenance: string; + occurredAt: string; + createdAt: string; + /** Resource tags for grant post-filter (same as search). */ + accessTags: string[]; + createdByPrincipalId: string | null; +}; + +export type FeedResult = { + entries: FeedEntry[]; + /** Highest feedSeq in this page (consumer stores as next `after`). */ + nextCursor: number | null; +}; + +export class FeedInputError extends Error { + readonly status = 400; + constructor(message: string) { + super(message); + this.name = "FeedInputError"; + } +} + +export async function fetchFeed( + db: Db, + args: FeedArgs, +): Promise { + const after = Math.max(0, Math.floor(args.after ?? 0)); + const limit = Math.min( + FEED_LIMIT_MAX, + Math.max(FEED_LIMIT_MIN, Math.floor(args.limit ?? FEED_LIMIT_DEFAULT)), + ); + const exclude = args.excludeGenerator?.trim() || undefined; + + const conditions = [ + eq(memoryVersion.tenantId, args.tenantId), + eq(memoryVersion.generation, LIVE_GENERATION), + // Live feed: active or superseded commits only. Deprecated / tombstoned + // versions are retention write-path outcomes and stay out of the cursor stream. + sql`${memoryVersion.status} IN ('active', 'superseded')`, + gt(memoryVersion.feedSeq, after), + ]; + + if (exclude) { + conditions.push( + or( + isNull(memoryVersion.generatorAgentId), + ne(memoryVersion.generatorAgentId, exclude), + )!, + ); + } + + const rows = await db + .select({ + feedSeq: memoryVersion.feedSeq, + versionId: memoryVersion.id, + documentId: memoryVersion.documentId, + kind: memoryDocument.kind, + title: memoryDocument.title, + status: memoryVersion.status, + createdByKind: memoryVersion.createdByKind, + generatorAgentId: memoryVersion.generatorAgentId, + provenance: memoryVersion.provenance, + occurredAt: memoryVersion.occurredAt, + createdAt: memoryVersion.ingestedAt, + accessTags: memoryDocument.accessTags, + createdByPrincipalId: memoryVersion.createdByPrincipalId, + }) + .from(memoryVersion) + .innerJoin( + memoryDocument, + eq(memoryDocument.id, memoryVersion.documentId), + ) + .where(and(...conditions)) + .orderBy(asc(memoryVersion.feedSeq)) + .limit(limit); + + const entries: FeedEntry[] = rows.map((r) => ({ + feedSeq: Number(r.feedSeq), + versionId: r.versionId, + documentId: r.documentId, + kind: r.kind, + title: r.title, + status: r.status, + createdByKind: r.createdByKind, + generatorAgentId: r.generatorAgentId, + provenance: r.provenance, + occurredAt: r.occurredAt.toISOString(), + createdAt: r.createdAt.toISOString(), + accessTags: (r.accessTags as string[] | null) ?? [], + createdByPrincipalId: r.createdByPrincipalId, + })); + + const nextCursor = + entries.length > 0 ? entries[entries.length - 1]!.feedSeq : null; + + return { entries, nextCursor }; +} + +/** + * Apply a document-access filter to a raw feed page. + * + * **Cursor rule:** `nextCursor` always comes from the raw examined page, + * not from the last allowed entry. A fully denied page must still advance + * the consumer past those `feed_seq` values or the poll stalls forever. + */ +export function feedPageAfterAccessFilter( + raw: { entries: readonly T[]; nextCursor: number | null }, + allowed: readonly T[], +): { entries: T[]; nextCursor: number | null } { + return { + entries: [...allowed], + nextCursor: raw.nextCursor, + }; +} diff --git a/src/services/retention.test.ts b/src/services/retention.test.ts new file mode 100644 index 0000000..8d73921 --- /dev/null +++ b/src/services/retention.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "bun:test"; + +// Pure policy helpers exercised via types + export surface. +// DB mutation tests need a real postgres; unit coverage is the lockstep +// enum + this smoke import. + +import { + deprecateVersion, + hardDeleteDocument, + setRetentionClass, + sweepEphemeral, + tombstoneDocument, +} from "./retention.ts"; + +describe("retention exports", () => { + it("exposes the CL-5871 write verbs", () => { + expect(typeof deprecateVersion).toBe("function"); + expect(typeof tombstoneDocument).toBe("function"); + expect(typeof hardDeleteDocument).toBe("function"); + expect(typeof sweepEphemeral).toBe("function"); + expect(typeof setRetentionClass).toBe("function"); + }); +}); diff --git a/src/services/retention.ts b/src/services/retention.ts new file mode 100644 index 0000000..719f4d7 --- /dev/null +++ b/src/services/retention.ts @@ -0,0 +1,203 @@ +/** + * Retention write paths (CL-5871). + * See docs/RETENTION.md. + */ +import { and, eq, inArray, isNotNull, lt, or, sql } from "drizzle-orm"; + +import type { Db } from "../db/client.ts"; +import { memoryChunk, memoryDocument, memoryVersion } from "../db/schema.ts"; +import type { RetentionClass } from "../core/enums.ts"; + +export type RetentionMutationResult = { + versionId: string; + documentId: string; + status: string; +}; + +export async function deprecateVersion( + db: Db, + input: { + tenantId: string; + versionId: string; + reason?: string; + }, +): Promise { + const now = new Date(); + const updated = await db + .update(memoryVersion) + .set({ + status: "deprecated", + deprecatedAt: now, + deprecatedReason: input.reason ?? "deprecated", + }) + .where( + and( + eq(memoryVersion.tenantId, input.tenantId), + eq(memoryVersion.id, input.versionId), + inArray(memoryVersion.status, ["active", "superseded"]), + ), + ) + .returning({ + versionId: memoryVersion.id, + documentId: memoryVersion.documentId, + status: memoryVersion.status, + }); + return updated[0] ?? null; +} + +/** + * Tombstone: hide from search/feed, redact chunk text, keep row for audit. + * Applies to the document's live active (or deprecated) versions. + */ +export async function tombstoneDocument( + db: Db, + input: { + tenantId: string; + documentId: string; + reason?: string; + }, +): Promise<{ versions: number }> { + const now = new Date(); + const versions = await db + .update(memoryVersion) + .set({ + status: "tombstoned", + deprecatedAt: now, + deprecatedReason: input.reason ?? "tombstoned", + }) + .where( + and( + eq(memoryVersion.tenantId, input.tenantId), + eq(memoryVersion.documentId, input.documentId), + inArray(memoryVersion.status, ["active", "deprecated", "superseded"]), + ), + ) + .returning({ id: memoryVersion.id }); + + if (versions.length > 0) { + await db + .update(memoryChunk) + .set({ text: "[redacted]" }) + .where( + and( + eq(memoryChunk.tenantId, input.tenantId), + eq(memoryChunk.documentId, input.documentId), + ), + ); + } + return { versions: versions.length }; +} + +/** + * Hard-delete a document (cascade chunks/versions/edges via FKs where set). + * Blocked for durable retention_class on any non-tombstoned version. + */ +export async function hardDeleteDocument( + db: Db, + input: { + tenantId: string; + documentId: string; + }, +): Promise<{ deleted: boolean; reason?: string }> { + const durable = await db + .select({ id: memoryVersion.id }) + .from(memoryVersion) + .where( + and( + eq(memoryVersion.tenantId, input.tenantId), + eq(memoryVersion.documentId, input.documentId), + eq(memoryVersion.retentionClass, "durable"), + sql`${memoryVersion.status} <> 'tombstoned'`, + ), + ) + .limit(1); + + if (durable.length > 0) { + return { + deleted: false, + reason: "document has durable retention_class versions; tombstone first", + }; + } + + const deleted = await db + .delete(memoryDocument) + .where( + and( + eq(memoryDocument.tenantId, input.tenantId), + eq(memoryDocument.id, input.documentId), + ), + ) + .returning({ id: memoryDocument.id }); + + return { deleted: deleted.length > 0 }; +} + +/** + * Sweep ephemeral versions past valid_until (or 7d default from ingested_at). + * Auto-deprecates expired ephemeral versions (core stays cron-free — host + * schedules this). Hard-delete remains an explicit separate verb. + */ +export async function sweepEphemeral( + db: Db, + input: { + tenantId: string; + now?: Date; + }, +): Promise<{ versionsDeprecated: number }> { + const now = input.now ?? new Date(); + const expired = await db + .update(memoryVersion) + .set({ + status: "deprecated", + deprecatedAt: now, + deprecatedReason: "ephemeral_ttl", + }) + .where( + and( + eq(memoryVersion.tenantId, input.tenantId), + eq(memoryVersion.retentionClass, "ephemeral"), + eq(memoryVersion.status, "active"), + or( + and( + isNotNull(memoryVersion.validUntil), + lt(memoryVersion.validUntil, now), + ), + and( + sql`${memoryVersion.validUntil} IS NULL`, + lt( + memoryVersion.ingestedAt, + new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000), + ), + ), + ), + ), + ) + .returning({ id: memoryVersion.id }); + + return { versionsDeprecated: expired.length }; +} + +export async function setRetentionClass( + db: Db, + input: { + tenantId: string; + versionId: string; + retentionClass: RetentionClass; + }, +): Promise { + const updated = await db + .update(memoryVersion) + .set({ retentionClass: input.retentionClass }) + .where( + and( + eq(memoryVersion.tenantId, input.tenantId), + eq(memoryVersion.id, input.versionId), + ), + ) + .returning({ + versionId: memoryVersion.id, + documentId: memoryVersion.documentId, + status: memoryVersion.status, + }); + return updated[0] ?? null; +} diff --git a/src/services/search.test.ts b/src/services/search.test.ts index e43cae9..0fa6d87 100644 --- a/src/services/search.test.ts +++ b/src/services/search.test.ts @@ -6,6 +6,7 @@ import { fetchDenseCandidates, hnswEfSearch, snippet, + toHit, type CandidateRow, } from "./search.ts"; @@ -26,6 +27,8 @@ function candidate(overrides: Partial = {}): CandidateRow { rank: 1, occurredAt: new Date("2026-01-01T00:00:00Z"), authority: 0.5, + temporalClass: "event", + validUntil: null, ...overrides, }; } @@ -132,25 +135,23 @@ describe("deriveHybridEvidence", () => { expect(deriveHybridEvidence(lexicalRows, 1)).toBe("weak"); }); - // The bug this fix closes: a query resolved mostly through the DENSE - // channel has a low (or zero) lexical ts_rank, so before this fix - // deriveHybridEvidence always fell through to deriveEvidence(lexicalRows) - // and reported "weak" — even when the cross-encoder rerank was highly - // confident and authority was high. The reranked-path floor now reports - // "strong" in that case instead. - it("reports 'strong' when reranked + high rerank score + high authority, even though lexical ts_rank is low", () => { + // Living relevancy (CL-5867): strong also needs the corroboration gate — + // stated human OR supports ≥ floor. High authority alone is not enough. + it("reports 'strong' when reranked + high rerank score + high authority + supports, even though lexical ts_rank is low", () => { const lowLexicalRows = [candidate({ rank: 0.001, authority: 0.9 })]; const evidence = deriveHybridEvidence(lowLexicalRows, 1, { rerankScore: 0.85, authority: 0.9, + supports: 2, }); expect(evidence).toBe("strong"); }); - it("reports 'strong' even with NO lexical rows at all, given a confident reranked top hit", () => { + it("reports 'strong' even with NO lexical rows at all, given a confident reranked top hit with supports", () => { const evidence = deriveHybridEvidence([], 1, { rerankScore: 0.85, authority: 0.9, + supports: 2, }); expect(evidence).toBe("strong"); }); @@ -159,6 +160,7 @@ describe("deriveHybridEvidence", () => { const evidence = deriveHybridEvidence([], 1, { rerankScore: 0.2, authority: 0.9, + supports: 5, }); expect(evidence).toBe("weak"); }); @@ -167,12 +169,41 @@ describe("deriveHybridEvidence", () => { const evidence = deriveHybridEvidence([], 1, { rerankScore: 0.9, authority: 0.1, + supports: 5, }); expect(evidence).toBe("weak"); }); + it("reports 'weak' when high score/authority but no corroboration gate (no supports, not stated human)", () => { + const evidence = deriveHybridEvidence([], 1, { + rerankScore: 0.9, + authority: 0.9, + supports: 0, + provenance: "inferred", + createdByKind: "agent", + }); + expect(evidence).toBe("weak"); + }); + + it("reports 'strong' for stated human without supports when score floors clear", () => { + const evidence = deriveHybridEvidence([], 1, { + rerankScore: 0.85, + authority: 0.9, + supports: 0, + provenance: "stated", + createdByKind: "human", + }); + expect(evidence).toBe("strong"); + }); + it("falls back to the lexical evidence path when reranking did not run (no rerankedTop)", () => { - const strongLexicalRows = [candidate({ rank: 0.9, authority: 0.9 })]; + const strongLexicalRows = [ + candidate({ + rank: 0.9, + authority: 0.9, + supports: 2, + }), + ]; expect(deriveHybridEvidence(strongLexicalRows, 1)).toBe("strong"); }); }); @@ -231,8 +262,7 @@ describe("fetchDenseCandidates hnsw tuning", () => { unsafe: (sqlText: string) => { statements.push(sqlText); return Promise.resolve( - sqlText.includes('FROM "memory"."embed_model"') || - sqlText.includes("FROM memory_embed_model") + sqlText.includes('FROM "memory"."embed_model"') ? [MODEL_ROW] : [], ); @@ -374,6 +404,7 @@ describe("fetchDenseCandidates kind/entity filtering", () => { unsafe: (sqlText: string, params?: unknown[]) => Promise; savepoint: (fn: (sp: FakeTx) => Promise) => Promise; }; + const statements: string[] = []; function evaluate(sqlText: string, params: unknown[]): unknown[] { let rows = DENSE_ROWS; const kindMatch = sqlText.match(/kd\.kind = ANY\(\$(\d+)/); @@ -394,6 +425,7 @@ describe("fetchDenseCandidates kind/entity filtering", () => { } const tx: FakeTx = { unsafe: (sqlText: string, params: unknown[] = []) => { + statements.push(sqlText); if (sqlText.includes("ORDER BY")) { return Promise.resolve(evaluate(sqlText, params)); } @@ -402,18 +434,21 @@ describe("fetchDenseCandidates kind/entity filtering", () => { savepoint: (fn: (sp: FakeTx) => Promise) => fn(tx), }; const rawSql = { - unsafe: (sqlText: string) => - Promise.resolve( - // CL-5233 qualified the table; keep the pre-qualify form so an - // accidental revert still fails this suite the same way. - sqlText.includes('FROM "memory"."embed_model"') || - sqlText.includes("FROM memory_embed_model") + unsafe: (sqlText: string) => { + statements.push(sqlText); + return Promise.resolve( + // CL-5233 qualified the table — only the fully-qualified form matches. + sqlText.includes('FROM "memory"."embed_model"') ? [MODEL_ROW] : [], - ), + ); + }, begin: (cb: (t: FakeTx) => Promise) => cb(tx), }; - return rawSql as unknown as Parameters[0]["sql"]; + return { + rawSql: rawSql as unknown as Parameters[0]["sql"], + statements, + }; } function baseArgs(sql: Parameters[0]["sql"]) { @@ -433,8 +468,9 @@ describe("fetchDenseCandidates kind/entity filtering", () => { } it("excludes a semantically-similar chunk whose document kind does not match `kinds`", async () => { + const fake = fakeRawSql(); const rows = await fetchDenseCandidates({ - ...baseArgs(fakeRawSql()), + ...baseArgs(fake.rawSql), kinds: ["task"], }); const chunkIds = rows?.map((r) => r.chunkId) ?? []; @@ -443,8 +479,9 @@ describe("fetchDenseCandidates kind/entity filtering", () => { }); it("excludes a semantically-similar chunk whose document is not linked to any requested entityId", async () => { + const fake = fakeRawSql(); const rows = await fetchDenseCandidates({ - ...baseArgs(fakeRawSql()), + ...baseArgs(fake.rawSql), entityIds: ["e-match"], }); const chunkIds = rows?.map((r) => r.chunkId) ?? []; @@ -452,16 +489,32 @@ describe("fetchDenseCandidates kind/entity filtering", () => { expect(chunkIds).not.toContain("chunk-note"); }); + it("entity filter targets memory.edge (not pre-rename knowledge_edge)", async () => { + const fake = fakeRawSql(); + await fetchDenseCandidates({ + ...baseArgs(fake.rawSql), + entityIds: ["e-match"], + }); + const denseSelect = fake.statements.find( + (s) => s.includes("ORDER BY") && s.includes("ke."), + ); + expect(denseSelect).toBeDefined(); + expect(denseSelect).toContain('FROM "memory"."edge" ke'); + expect(denseSelect).not.toContain("knowledge_edge"); + }); + it("applies no kind/entity predicate — and returns every semantically-similar chunk — when neither filter is provided", async () => { - const rows = await fetchDenseCandidates(baseArgs(fakeRawSql())); + const fake = fakeRawSql(); + const rows = await fetchDenseCandidates(baseArgs(fake.rawSql)); const chunkIds = rows?.map((r) => r.chunkId) ?? []; expect(chunkIds).toContain("chunk-task"); expect(chunkIds).toContain("chunk-note"); }); it("treats an empty kinds/entityIds array as no filter, same as lexical", async () => { + const fake = fakeRawSql(); const rows = await fetchDenseCandidates({ - ...baseArgs(fakeRawSql()), + ...baseArgs(fake.rawSql), kinds: [], entityIds: [], }); @@ -470,3 +523,69 @@ describe("fetchDenseCandidates kind/entity filtering", () => { expect(chunkIds).toContain("chunk-note"); }); }); + +describe("toHit — wire attribution (CL-5870)", () => { + it("surfaces provenance, temporal, corroboration, and derived_from on the hit", () => { + const hit = toHit( + candidate({ + provenance: "inferred", + sourceClass: "derived", + temporalClass: "state", + validUntil: new Date("2026-12-01T00:00:00Z"), + supports: 2, + contradicts: 0, + derivedFrom: ["kv_source_1", "kv_source_2"], + generatorAgentId: "resident-distiller", + createdByKind: "agent", + }), + ); + expect(hit.version_id).toBe("ver_1"); + expect(hit.provenance).toBe("inferred"); + expect(hit.source_class).toBe("derived"); + expect(hit.temporal_class).toBe("state"); + expect(hit.valid_until).toBe("2026-12-01T00:00:00.000Z"); + expect(hit.supports).toBe(2); + expect(hit.contradicts).toBe(0); + expect(hit.derived_from).toEqual(["kv_source_1", "kv_source_2"]); + expect(hit.generator_agent_id).toBe("resident-distiller"); + expect(hit.created_by_kind).toBe("agent"); + }); + + it("omits optional attribution fields when absent (additive wire)", () => { + const hit = toHit(candidate()); + expect(hit.provenance).toBeUndefined(); + expect(hit.source_class).toBeUndefined(); + expect(hit.derived_from).toBeUndefined(); + expect(hit.generator_agent_id).toBeUndefined(); + expect(hit.supports).toBe(0); + expect(hit.contradicts).toBe(0); + expect(hit.temporal_class).toBe("event"); + }); + + it("distinguishes stated human vs inferred agent attribution shapes", () => { + const human = toHit( + candidate({ + provenance: "stated", + sourceClass: "native", + createdByKind: "human", + generatorAgentId: null, + }), + ); + const distilled = toHit( + candidate({ + provenance: "inferred", + sourceClass: "derived", + createdByKind: "agent", + generatorAgentId: "resident-distiller", + derivedFrom: ["kv_raw"], + }), + ); + expect(human.provenance).toBe("stated"); + expect(human.created_by_kind).toBe("human"); + expect(human.generator_agent_id).toBeUndefined(); + expect(distilled.provenance).toBe("inferred"); + expect(distilled.created_by_kind).toBe("agent"); + expect(distilled.generator_agent_id).toBe("resident-distiller"); + expect(distilled.derived_from).toEqual(["kv_raw"]); + }); +}); diff --git a/src/services/search.ts b/src/services/search.ts index 636df0a..b7f93b3 100644 --- a/src/services/search.ts +++ b/src/services/search.ts @@ -11,8 +11,11 @@ import { import { createRawSqlClient } from "../core/embed-sql.ts"; import { cosineDistanceExpr, + computeModelKey, EMBED_TABLE_NAME_PATTERN, resolveActiveEmbedTable, + resolveEmbedTableByModelKey, + type ActiveEmbedTable, } from "../core/embed-model-registry.ts"; import { embedTexts, type EmbedClientConfig } from "../core/embed-client.ts"; import { @@ -34,11 +37,17 @@ import { DEFAULT_OVERFETCH_MULTIPLIER, fuseRrf, normalizeScoresToUnit, - recencyBoostMultiplier, + temporalRecencyMultiplier, RECENCY_HALF_LIFE_MS, toRankedCandidates, type DegradeFlag, } from "../core/hybrid-search.ts"; +import { + corroborationFactor, + effectiveAuthority, + meetsStrongEvidenceGate, + type CorroborationCounts, +} from "../core/corroboration.ts"; import { formatCaughtError, log } from "../log.ts"; import { resolveGenerationSearchParams } from "./transform.ts"; import type { @@ -87,8 +96,9 @@ export function authorityWeightedScore( } // Evidence cap: a hit only reaches 'strong' when BOTH its raw lexical -// relevance clears STRONG_RANK_FLOOR AND its authority clears this floor. -const AUTHORITY_STRONG_FLOOR = 0.3; +// relevance clears STRONG_RANK_FLOOR AND the strong-evidence gate (authority +// floor + stated human OR corroboration floor) clears. See docs/RELEVANCY.md. +export const AUTHORITY_STRONG_FLOOR = 0.3; // Second, reranked-path-only strong floor. `deriveHybridEvidence`'s lexical // ts_rank check under-reports a query resolved mostly through the DENSE @@ -126,8 +136,21 @@ export interface CandidateRow { rank: number; occurredAt: Date; // The version's stored 0..1 authority score (computed at capture time; - // never recomputed here). + // never recomputed here). Ranking multiplies by corroborationFactor. authority: number; + // Temporal ranking class + validity (docs/TEMPORAL.md). Defaults applied + // when a row predates the temporal migration (should not happen after 0004). + temporalClass: "event" | "deadline" | "state" | "lesson"; + validUntil: Date | null; + /** Lineage class (native | imported | derived) for attribution. */ + sourceClass?: string; + /** Provenance mode for evidence gating (stated | inferred | unknown). */ + provenance?: string; + /** Independent supports / contradicts targeting this version (search-time). */ + supports?: number; + contradicts?: number; + /** Source version ids via derived_from edges (CL-5870). */ + derivedFrom?: string[]; } export function snippet(text: string, maxLen = 240): string { @@ -175,7 +198,7 @@ export function toHit( // authority handling baked in (the reranked path's boosted score). authorityWeight: number = AUTHORITY_WEIGHT, ): SearchHit { - return { + const hit: SearchHit = { chunk_id: row.chunkId, document_id: row.documentId, version: row.version, @@ -183,14 +206,18 @@ export function toHit( status: row.status, score: scoreOverride ?? - authorityWeightedScore(row.rank, row.authority, authorityWeight), + authorityWeightedScore( + row.rank, + effectiveAuthority(row.authority, { + supports: row.supports ?? 0, + contradicts: row.contradicts ?? 0, + }), + authorityWeight, + ), title: row.title, snippet: snippet(row.snippetText), kind: row.kind, created_by_kind: row.createdByKind, - ...(row.generatorAgentId - ? { generator_agent_id: row.generatorAgentId } - : {}), citation: { adapter: row.adapter, external_ref: row.externalRef, @@ -198,20 +225,45 @@ export function toHit( }, entity_ids: [], channels_matched: channelsMatched, + temporal_class: row.temporalClass, + occurred_at: row.occurredAt.toISOString(), + valid_until: row.validUntil ? row.validUntil.toISOString() : null, + supports: row.supports ?? 0, + contradicts: row.contradicts ?? 0, }; + if (row.generatorAgentId) { + hit.generator_agent_id = row.generatorAgentId; + } + if (row.provenance !== undefined) { + hit.provenance = row.provenance as NonNullable; + } + if (row.sourceClass !== undefined) { + hit.source_class = row.sourceClass as NonNullable; + } + if (row.derivedFrom !== undefined) { + hit.derived_from = row.derivedFrom; + } + return hit; } -// Evidence is capped by authority, not just relevance: the top-ranked hit -// (by raw relevance) must ALSO clear AUTHORITY_STRONG_FLOOR to report -// 'strong'. A relevant hit backed only by a low-authority source reports -// 'weak' instead of overstating confidence. +// Evidence is capped by the strong-evidence gate (authority + corroboration / +// stated human), not relevance alone. A relevant hit with low authority or +// only inferred single-source content reports 'weak'. export function deriveEvidence( hits: readonly CandidateRow[], ): SearchResponse["evidence"] { if (hits.length === 0) return "none"; const top = hits.reduce((best, h) => (h.rank > best.rank ? h : best)); if (top.rank < STRONG_RANK_FLOOR) return "weak"; - return top.authority >= AUTHORITY_STRONG_FLOOR ? "strong" : "weak"; + return meetsStrongEvidenceGate({ + authority: top.authority, + supports: top.supports ?? 0, + provenance: top.provenance, + createdByKind: top.createdByKind, + authorityFloor: AUTHORITY_STRONG_FLOOR, + }) + ? "strong" + : "weak"; } // Evidence is primarily derived from the LEXICAL channel, using the same @@ -221,22 +273,34 @@ export function deriveEvidence( // against fused scores would make "strong" unreachable via that path. A // SECOND, independent "strong" path exists for the reranked path: when // `rerankedTop` is supplied (reranking ran and produced a top hit), a rerank -// score clearing RERANK_STRONG_FLOOR combined with authority clearing -// AUTHORITY_STRONG_FLOOR is strong evidence on its own, even when the -// lexical channel barely (or never) matched — this is what lets a query -// resolved mostly through the DENSE channel report "strong" instead of -// always "weak". A result that came back only through the dense channel, -// on the non-reranked/degraded path (no `rerankedTop`), is still "weak". +// score clearing RERANK_STRONG_FLOOR combined with the strong-evidence gate +// is strong evidence on its own, even when the lexical channel barely (or +// never) matched — this is what lets a query resolved mostly through the +// DENSE channel report "strong" instead of always "weak". A result that +// came back only through the dense channel, on the non-reranked/degraded +// path (no `rerankedTop`), is still "weak". export function deriveHybridEvidence( lexicalRows: readonly CandidateRow[], finalHitCount: number, - rerankedTop?: { rerankScore: number; authority: number }, + rerankedTop?: { + rerankScore: number; + authority: number; + supports?: number; + provenance?: string; + createdByKind?: string; + }, ): SearchResponse["evidence"] { if (finalHitCount === 0) return "none"; if ( rerankedTop && rerankedTop.rerankScore >= RERANK_STRONG_FLOOR && - rerankedTop.authority >= AUTHORITY_STRONG_FLOOR + meetsStrongEvidenceGate({ + authority: rerankedTop.authority, + supports: rerankedTop.supports ?? 0, + provenance: rerankedTop.provenance, + createdByKind: rerankedTop.createdByKind, + authorityFloor: AUTHORITY_STRONG_FLOOR, + }) ) { return "strong"; } @@ -259,7 +323,14 @@ export function dedupeCandidatesPerDocument( ): CandidateRow[] { const scoreOf = (row: CandidateRow): number => applyAuthorityPrior - ? authorityWeightedScore(row.rank, row.authority, authorityWeight) + ? authorityWeightedScore( + row.rank, + effectiveAuthority(row.authority, { + supports: row.supports ?? 0, + contradicts: row.contradicts ?? 0, + }), + authorityWeight, + ) : row.rank; const byDocument = new Map(); @@ -306,6 +377,89 @@ export async function attachEntityIds( return map; } +/** + * Batch-load supports/contradicts counts for candidate version ids. + * Edges target `to_type = version`. Mutates rows in place. + */ +export async function attachCorroborationCounts( + db: Db, + tenantId: string, + rows: CandidateRow[], +): Promise { + const versionIds = [...new Set(rows.map((r) => r.versionId))]; + if (versionIds.length === 0) return; + + const edges = await db + .select({ + toRef: memoryEdge.toRef, + rel: memoryEdge.rel, + }) + .from(memoryEdge) + .where( + and( + eq(memoryEdge.tenantId, tenantId), + eq(memoryEdge.toType, "version"), + inArray(memoryEdge.toRef, versionIds), + inArray(memoryEdge.rel, ["supports", "contradicts"]), + ), + ); + + const counts = new Map(); + for (const id of versionIds) { + counts.set(id, { supports: 0, contradicts: 0 }); + } + for (const edge of edges) { + const c = counts.get(edge.toRef) ?? { supports: 0, contradicts: 0 }; + if (edge.rel === "supports") c.supports += 1; + else if (edge.rel === "contradicts") c.contradicts += 1; + counts.set(edge.toRef, c); + } + for (const row of rows) { + const c = counts.get(row.versionId) ?? { supports: 0, contradicts: 0 }; + row.supports = c.supports; + row.contradicts = c.contradicts; + } +} + +/** + * Load derived_from edges (from hit version → source version) for attribution. + * Does not grant-filter sources; the DocumentStore / host may strip inaccessible + * source ids after the security post-filter if needed. + */ +export async function attachDerivedFrom( + db: Db, + tenantId: string, + rows: CandidateRow[], +): Promise { + if (rows.length === 0) return; + const versionIds = [...new Set(rows.map((r) => r.versionId))]; + const edges = await db + .select({ + fromRef: memoryEdge.fromRef, + toRef: memoryEdge.toRef, + }) + .from(memoryEdge) + .where( + and( + eq(memoryEdge.tenantId, tenantId), + eq(memoryEdge.fromType, "version"), + eq(memoryEdge.toType, "version"), + eq(memoryEdge.rel, "derived_from"), + inArray(memoryEdge.fromRef, versionIds), + ), + ); + + const map = new Map(); + for (const e of edges) { + const list = map.get(e.fromRef) ?? []; + list.push(e.toRef); + map.set(e.fromRef, list); + } + for (const row of rows) { + row.derivedFrom = map.get(row.versionId) ?? []; + } +} + // The kinds/entityIds shape shared by both channels' candidate-query params // and by HybridSearchArgs (which fans a single caller-supplied pair of these // out to both channels before fusion). An empty array is treated identically @@ -326,6 +480,8 @@ interface LexicalCandidateParams extends ChannelFilterFields { // a replayed generation's chunks never leak into a live search and vice // versa (the replay pipeline). generation?: string | undefined; + /** When true, include deprecated versions alongside active (CL-5871). */ + includeDeprecated?: boolean | undefined; } // The single FTS-candidate query for the lexical channel. Returns raw, @@ -344,11 +500,14 @@ export async function fetchLexicalCandidates( kinds, entityIds, generation = LIVE_GENERATION, + includeDeprecated = false, } = params; const conditions = [ eq(memoryChunk.tenantId, tenantId), - eq(memoryVersion.status, "active"), + includeDeprecated + ? inArray(memoryVersion.status, ["active", "deprecated"]) + : eq(memoryVersion.status, "active"), eq(memoryVersion.generation, generation), ]; @@ -399,6 +558,10 @@ export async function fetchLexicalCandidates( rank: rankExpr, occurredAt: memoryVersion.occurredAt, authority: memoryVersion.authority, + temporalClass: memoryVersion.temporalClass, + validUntil: memoryVersion.validUntil, + provenance: memoryVersion.provenance, + sourceClass: memoryVersion.sourceClass, }) .from(memoryChunk) .innerJoin( @@ -424,16 +587,13 @@ interface FetchDenseCandidatesArgs extends ChannelFilterFields { principalId: string | null; query: string; overfetchLimit: number; - // Defaults to 'live' — see fetchLexicalCandidates' generation note. NOTE: - // this filters the chunk/version join, not which per-model embedding - // TABLE is queried — resolveActiveEmbedTable picks the tenant's single - // most-recently-activated model regardless of generation, so a replay run - // that activates a DIFFERENT embed model than the live one currently uses - // becomes the tenant's active table for every generation's dense channel, - // including live's. Scoping activation itself per-generation is out of - // the replay pipeline's scope; callers should reuse the live embed model in a - // transform_config unless they intend that tradeoff. + // Defaults to 'live' — see fetchLexicalCandidates' generation note. + // Dense table resolution is generation-aware: live uses the tenant's + // active embed model; a replay generation uses the model_key implied by + // the embed client config passed in (from that run's transform_config), + // which may be only `ready` and must not require status='active'. generation?: string | undefined; + includeDeprecated?: boolean | undefined; } // Whether this pool's pgvector understands hnsw.iterative_scan, learned @@ -473,12 +633,26 @@ export async function fetchDenseCandidates( kinds, entityIds, generation = LIVE_GENERATION, + includeDeprecated = false, } = args; if (query === "") return null; const embedSqlClient = createRawSqlClient(rawSql); - const activeTable = await resolveActiveEmbedTable(embedSqlClient, tenantId); + let activeTable: ActiveEmbedTable | null; + if (generation === LIVE_GENERATION) { + activeTable = await resolveActiveEmbedTable(embedSqlClient, tenantId); + } else { + const modelKey = computeModelKey( + embedClientConfig.baseUrl, + embedClientConfig.modelId, + ); + activeTable = await resolveEmbedTableByModelKey( + embedSqlClient, + tenantId, + modelKey, + ); + } if (!activeTable) return null; if (!EMBED_TABLE_NAME_PATTERN.test(activeTable.tableName)) { @@ -516,7 +690,7 @@ export async function fetchDenseCandidates( if (entityIds && entityIds.length > 0) { params.push(entityIds); entityClause = `AND kd.id IN ( - SELECT ke.from_ref FROM memory_edge ke + SELECT ke.from_ref FROM "memory"."edge" ke WHERE ke.tenant_id = $1 AND ke.from_type = 'document' AND ke.to_type = 'entity' AND ke.to_ref = ANY($${params.length}::text[]) )`; @@ -527,12 +701,15 @@ export async function fetchDenseCandidates( kv.version AS version, kv.status AS status, kd.title AS title, kd.kind AS kind, kd.adapter AS adapter, kd.external_ref AS external_ref, kv.created_by_kind AS created_by_kind, kv.generator_agent_id AS generator_agent_id, - c.text AS snippet_text, kv.occurred_at AS occurred_at, kv.authority AS authority + c.text AS snippet_text, kv.occurred_at AS occurred_at, kv.authority AS authority, + kv.temporal_class AS temporal_class, kv.valid_until AS valid_until, + kv.provenance AS provenance, kv.source_class AS source_class FROM ${activeTable.tableName} e JOIN "memory"."chunk" c ON c.id = e.chunk_id JOIN "memory"."version" kv ON kv.id = c.version_id JOIN "memory"."document" kd ON kd.id = c.document_id - WHERE e.tenant_id = $1 AND c.tenant_id = $1 AND kv.status = 'active' + WHERE e.tenant_id = $1 AND c.tenant_id = $1 + AND kv.status ${includeDeprecated ? "IN ('active', 'deprecated')" : "= 'active'"} AND kv.generation = ${generationParam} ${kindClause} ${entityClause} @@ -570,42 +747,70 @@ export async function fetchDenseCandidates( return tx.unsafe(sqlText, params as never[]); }); - return (rows as unknown as Array>).map((row) => ({ - chunkId: row["chunk_id"] as string, - documentId: row["document_id"] as string, - versionId: row["version_id"] as string, - version: row["version"] as number, - status: row["status"] as CandidateRow["status"], - title: row["title"] as string, - kind: row["kind"] as string, - adapter: row["adapter"] as string, - externalRef: row["external_ref"] as string, - createdByKind: row["created_by_kind"] as CandidateRow["createdByKind"], - generatorAgentId: (row["generator_agent_id"] as string | null) ?? null, - snippetText: row["snippet_text"] as string, - // Dense candidates carry no ts_rank-comparable score; `rank` is - // overwritten with the fused RRF score once fusion runs, and is never - // read before that. - rank: 0, - occurredAt: new Date(row["occurred_at"] as string), - authority: row["authority"] as number, - })); + return (rows as unknown as Array>).map((row) => { + const sourceClass = row["source_class"] as string | null | undefined; + const provenance = row["provenance"] as string | null | undefined; + const base: CandidateRow = { + chunkId: row["chunk_id"] as string, + documentId: row["document_id"] as string, + versionId: row["version_id"] as string, + version: row["version"] as number, + status: row["status"] as CandidateRow["status"], + title: row["title"] as string, + kind: row["kind"] as string, + adapter: row["adapter"] as string, + externalRef: row["external_ref"] as string, + createdByKind: row["created_by_kind"] as CandidateRow["createdByKind"], + generatorAgentId: (row["generator_agent_id"] as string | null) ?? null, + snippetText: row["snippet_text"] as string, + // Dense candidates carry no ts_rank-comparable score; `rank` is + // overwritten with the fused RRF score once fusion runs, and is never + // read before that. + rank: 0, + occurredAt: new Date(row["occurred_at"] as string), + authority: row["authority"] as number, + temporalClass: + (row["temporal_class"] as CandidateRow["temporalClass"]) ?? "event", + validUntil: row["valid_until"] + ? new Date(row["valid_until"] as string) + : null, + }; + if (provenance != null && provenance !== "") { + base.provenance = provenance; + } + if (sourceClass != null && sourceClass !== "") { + base.sourceClass = sourceClass; + } + return base; + }); } -// Vectors for the MMR diversity pass, pulled from the tenant's ACTIVE -// per-model embedding table (never a superseded or inactive model's -// table). `pgvector` returns its column as text; the text form (`[1,2,3]`) -// is valid JSON, so `JSON.parse` is the exact inverse of the -// `JSON.stringify` the capture/embed pipeline writes on ingest. +// Vectors for the MMR diversity pass, pulled from the generation-scoped +// embedding table (active model for live; model_key for a replay generation). async function fetchChunkVectors( rawSql: RawSql, tenantId: string, chunkIds: readonly string[], + embedClientConfig: EmbedClientConfig, + generation: string = LIVE_GENERATION, ): Promise> { if (chunkIds.length === 0) return new Map(); const embedSqlClient = createRawSqlClient(rawSql); - const activeTable = await resolveActiveEmbedTable(embedSqlClient, tenantId); + let activeTable: ActiveEmbedTable | null; + if (generation === LIVE_GENERATION) { + activeTable = await resolveActiveEmbedTable(embedSqlClient, tenantId); + } else { + const modelKey = computeModelKey( + embedClientConfig.baseUrl, + embedClientConfig.modelId, + ); + activeTable = await resolveEmbedTableByModelKey( + embedSqlClient, + tenantId, + modelKey, + ); + } if (!activeTable) return new Map(); if (!EMBED_TABLE_NAME_PATTERN.test(activeTable.tableName)) { @@ -650,9 +855,30 @@ function applyBoosts( const normalized = normalizeScoresToUnit(rows.map((row) => row.rank)); return rows.map((row, index) => { const normScore = normalized[index] ?? 0; - const authorityMult = authorityBoostMultiplier(row.authority); - const recencyMult = recencyBoostMultiplier(row.occurredAt, now, recencyHalfLifeMs); - return { row, finalScore: normScore * authorityMult * recencyMult }; + const authorityMult = authorityBoostMultiplier( + effectiveAuthority(row.authority, { + supports: row.supports ?? 0, + contradicts: row.contradicts ?? 0, + }), + ); + // Corroboration also multiplies the final score once more via the same + // factor so a supported claim outranks an unsupported twin at equal + // authority snapshot (authorityBoost alone compresses high authorities). + const corrMult = corroborationFactor({ + supports: row.supports ?? 0, + contradicts: row.contradicts ?? 0, + }); + const recencyMult = temporalRecencyMultiplier({ + temporalClass: row.temporalClass, + occurredAt: row.occurredAt, + validUntil: row.validUntil, + now, + halfLifeMs: recencyHalfLifeMs, + }); + return { + row, + finalScore: normScore * authorityMult * corrMult * recencyMult, + }; }); } @@ -688,6 +914,8 @@ export interface HybridSearchArgs extends ChannelFilterFields { // instead, applying its transform_config's retrieval tuning when // resolvable (see resolveGenerationSearchParams, transform.ts). generation?: string | undefined; + /** Include deprecated versions in retrieval (default false). CL-5871. */ + includeDeprecated?: boolean | undefined; } const MS_PER_DAY = 24 * 60 * 60 * 1000; @@ -723,6 +951,7 @@ export async function hybridSearch( const { db, sql: rawSql, config, fetchImpl = fetch, now = new Date() } = deps; const { tenantId, principalId, kinds, entityIds } = args; const generation = args.generation ?? LIVE_GENERATION; + const includeDeprecated = args.includeDeprecated === true; const query = args.query.trim(); const k = Math.min( Math.max(1, Math.floor(args.k ?? DEFAULT_HYBRID_TOP_K)), @@ -743,7 +972,12 @@ export async function hybridSearch( const resolvedTuning = generation === LIVE_GENERATION ? null - : await resolveGenerationSearchParams(db, generation); + : await resolveGenerationSearchParams( + db, + generation, + config.embed, + tenantId, + ); const authorityWeight = resolvedTuning?.authorityWeight ?? AUTHORITY_WEIGHT; const recencyHalfLifeMs = @@ -769,9 +1003,11 @@ export async function hybridSearch( kinds, entityIds, generation, + includeDeprecated, }); - const embedClientConfig = toEmbedClientConfig(config.embed); + const embedClientConfig = + resolvedTuning?.embed ?? toEmbedClientConfig(config.embed); const rerankConfig = resolvedTuning?.rerank ?? toRerankClientConfig(config.rerank); let denseRows: CandidateRow[] = []; @@ -789,6 +1025,7 @@ export async function hybridSearch( kinds, entityIds, generation, + includeDeprecated, }); if (dense === null) { degraded = ["dense_unavailable"]; @@ -831,6 +1068,13 @@ export async function hybridSearch( mergedRows.push({ ...base, rank: candidate.score }); } + // Living relevancy: attach supports/contradicts before authority-weighted + // dedupe / boosts (capture-time authority snapshot stays on the row). + await attachCorroborationCounts(db, tenantId, mergedRows); + // Lexical-only evidence path also needs counts on the original channel rows. + await attachCorroborationCounts(db, tenantId, lexicalRows); + await attachDerivedFrom(db, tenantId, mergedRows); + let truncated: CandidateRow[]; // The final score per chunk, when the reranked path ran; absent on the // degraded/fallback path, where `toHit` falls back to its own @@ -887,6 +1131,8 @@ export async function hybridSearch( rawSql, tenantId, boosted.map((b) => b.row.chunkId), + embedClientConfig, + generation, ); const mmrItems: MmrItem[] = boosted.map((b) => ({ @@ -960,6 +1206,11 @@ export async function hybridSearch( ? { rerankScore: rawRerankScoreByChunk.get(topTruncated.chunkId) ?? 0, authority: topTruncated.authority, + supports: topTruncated.supports ?? 0, + ...(topTruncated.provenance !== undefined + ? { provenance: topTruncated.provenance } + : {}), + createdByKind: topTruncated.createdByKind, } : undefined; const evidence = deriveHybridEvidence(lexicalRows, hits.length, rerankedTop); diff --git a/src/services/share-grants.test.ts b/src/services/share-grants.test.ts new file mode 100644 index 0000000..57c9177 --- /dev/null +++ b/src/services/share-grants.test.ts @@ -0,0 +1,178 @@ +import { describe, expect, it } from "bun:test"; +import { authorize } from "@intx/authz"; + +import { + buildShareGrants, + documentTag, + materializeShareGrants, + MEMORY_SHARE_CONDITION_REGISTRY, + shareWidenReceipt, + splitAudienceWiden, +} from "./share-grants.ts"; +import { createInMemoryWritableGrantStore } from "../ports/writable-grant-store.ts"; +import { canAccessDocument } from "../grant-tags.ts"; + +describe("documentTag", () => { + it("scopes resource to the document id", () => { + expect(documentTag("kdoc_1")).toBe("memory.doc:kdoc_1"); + }); +}); + +describe("buildShareGrants", () => { + it("emits one allow/search grant per peer on the document tag", () => { + const grants = buildShareGrants({ + tenantId: "t1", + sharedByPrincipalId: "alice", + documentId: "kdoc_1", + sourceVersionId: "kver_1", + share: { principals: ["bob", "carol"] }, + }); + expect(grants).toHaveLength(2); + expect(grants.every((g) => g.resource === "memory.doc:kdoc_1")).toBe(true); + expect(grants.every((g) => g.action === "search" && g.effect === "allow")).toBe( + true, + ); + expect(grants.map((g) => g.principalId).sort()).toEqual(["bob", "carol"]); + expect(grants[0]?.conditions?.memoryShare).toEqual({ + sharedBy: "alice", + sourceVersionId: "kver_1", + documentId: "kdoc_1", + tenantId: "t1", + }); + expect(grants[0]?.origin).toBe("system"); + }); + + it("skips the sharer themselves and empty principals", () => { + const grants = buildShareGrants({ + tenantId: "t1", + sharedByPrincipalId: "alice", + documentId: "kdoc_1", + sourceVersionId: "kver_1", + share: { principals: ["alice", " ", "bob"] }, + }); + expect(grants).toHaveLength(1); + expect(grants[0]?.principalId).toBe("bob"); + }); + + it("returns empty when only tenant/tags sugar is set", () => { + const grants = buildShareGrants({ + tenantId: "t1", + sharedByPrincipalId: "alice", + documentId: "kdoc_1", + sourceVersionId: "kver_1", + share: { tenant: true, tags: ["memory.space:eng"] }, + }); + expect(grants).toHaveLength(0); + }); +}); + +describe("materializeShareGrants + canAccessDocument", () => { + it("peer can search after materialize; non-peer cannot", async () => { + const store = createInMemoryWritableGrantStore(); + await materializeShareGrants(store, { + tenantId: "t1", + sharedByPrincipalId: "alice", + documentId: "kdoc_1", + sourceVersionId: "kver_1", + share: { principals: ["bob"] }, + }); + + const tags = ["memory.owner:alice", documentTag("kdoc_1")]; + const registry = MEMORY_SHARE_CONDITION_REGISTRY; + + expect( + await canAccessDocument({ + grants: store, + tenantId: "t1", + principalId: "bob", + createdByPrincipalId: "alice", + accessTags: tags, + conditionRegistry: registry, + }), + ).toBe(true); + + expect( + await canAccessDocument({ + grants: store, + tenantId: "t1", + principalId: "eve", + createdByPrincipalId: "alice", + accessTags: tags, + conditionRegistry: registry, + }), + ).toBe(false); + + // Creator still allowed without a grant. + expect( + await canAccessDocument({ + grants: store, + tenantId: "t1", + principalId: "alice", + createdByPrincipalId: "alice", + accessTags: tags, + }), + ).toBe(true); + }); + + it("embargo: expired grant does not allow access", async () => { + const store = createInMemoryWritableGrantStore(); + await store.putGrant({ + id: "g_expired", + principalId: "bob", + resource: "memory.doc:kdoc_1", + action: "search", + effect: "allow", + origin: "system", + roleId: null, + expiresAt: new Date("2020-01-01T00:00:00Z"), + conditions: null, + }); + + const decision = await authorize( + store, + "bob", + "t1", + "memory.doc:kdoc_1", + "search", + ); + expect(decision.effect).toBe(null); + + expect( + await canAccessDocument({ + grants: store, + tenantId: "t1", + principalId: "bob", + createdByPrincipalId: "alice", + accessTags: [documentTag("kdoc_1")], + }), + ).toBe(false); + }); +}); + +describe("splitAudienceWiden", () => { + it("keeps source tags, flags new ones for approval", () => { + const { allowed, needsApproval } = splitAudienceWiden( + ["memory.owner:alice", "memory.space:eng"], + ["memory.owner:alice", "memory.space:eng", "memory.tenant:t1"], + ); + expect(allowed).toEqual(["memory.owner:alice", "memory.space:eng"]); + expect(needsApproval).toEqual(["memory.tenant:t1"]); + }); +}); + +describe("shareWidenReceipt", () => { + it("records approver, tags, and source version", () => { + const receipt = shareWidenReceipt({ + approvedBy: "alice", + tags: ["memory.tenant:t1"], + sourceVersionId: "kver_1", + approvedAt: new Date("2026-07-20T00:00:00Z"), + }); + expect(receipt).toEqual({ + approvedBy: "alice", + approvedAt: "2026-07-20T00:00:00.000Z", + tags: ["memory.tenant:t1"], + sourceVersionId: "kver_1", + }); + }); +}); diff --git a/src/services/share-grants.ts b/src/services/share-grants.ts new file mode 100644 index 0000000..2a88f31 --- /dev/null +++ b/src/services/share-grants.ts @@ -0,0 +1,161 @@ +/** + * Share materialization — tags alone are not grants (CL-5873). + * + * Locked decisions (Greybeard): + * - Approver default: **source owner** (not tenant admin). + * - Staging: **write-narrow-then-widen** (no pending_share status). + * - Ask-on-read: design-only in v1 (canAccessDocument still fail-closed on ask). + * + * Origin is constrained by `@intx/types` to system|role|creator|invoker — + * memory-share provenance lives in `conditions.memoryShare` (audit payload). + * Authz skips grants with non-null conditions unless a registry is provided; + * use `MEMORY_SHARE_CONDITION_REGISTRY` (merged automatically in resolveGrantConfig). + */ +import type { ConditionRegistry, GrantRule } from "@intx/authz"; +import { newId } from "../core/id.ts"; +import type { ShareSugar } from "../grant-tags.ts"; +import type { WritableGrantStore } from "../ports/writable-grant-store.ts"; + +/** Document-scoped resource tag for peer share grants. */ +export function documentTag(documentId: string): string { + return `memory.doc:${documentId}`; +} + +export const MEMORY_SHARE_CONDITION_KEY = "memoryShare"; + +/** + * Audit payload stored under conditions.memoryShare. + * The evaluator always returns true — this is provenance, not a gate. + */ +export type MemoryShareCondition = { + sharedBy: string; + sourceVersionId: string; + documentId: string; + tenantId: string; +}; + +/** + * Default registry so share grants with conditions are not fail-closed-skipped. + * Hosts may override the key; resolveGrantConfig merges host keys on top. + */ +export const MEMORY_SHARE_CONDITION_REGISTRY: ConditionRegistry = { + [MEMORY_SHARE_CONDITION_KEY]: () => true, +}; + +export type MaterializeShareGrantsInput = { + tenantId: string; + /** Principal who initiated the share (source owner / creator). */ + sharedByPrincipalId: string; + documentId: string; + /** Version that carried the share (for audit receipt). */ + sourceVersionId: string; + share: ShareSugar; +}; + +/** + * Build grant rules for peer principals on a document-scoped tag. + * + * - `share.principals`: one allow/search grant per peer on `memory.doc:`. + * - `share.tenant` / `share.tags`: do **not** auto-mint principal grants — + * those rely on host role/pattern grants already present on the tag. + * + * Returns rules only (does not write). Caller applies via WritableGrantStore. + */ +export function buildShareGrants( + input: MaterializeShareGrantsInput, +): GrantRule[] { + const peers = input.share.principals ?? []; + if (peers.length === 0) return []; + + const resource = documentTag(input.documentId); + const rules: GrantRule[] = []; + + for (const peer of peers) { + if (typeof peer !== "string" || peer.trim() === "") continue; + const principalId = peer.trim(); + // Never grant the owner to themselves via share — creator path already covers. + if (principalId === input.sharedByPrincipalId) continue; + + const sharePayload: MemoryShareCondition = { + sharedBy: input.sharedByPrincipalId, + sourceVersionId: input.sourceVersionId, + documentId: input.documentId, + tenantId: input.tenantId, + }; + + rules.push({ + id: newId("mgrt"), + principalId, + resource, + action: "search", + effect: "allow", + origin: "system", + roleId: null, + expiresAt: null, + // Single condition key so hosts only need one registry entry. + // Nested object carries audit provenance without extra evaluators. + conditions: { + [MEMORY_SHARE_CONDITION_KEY]: sharePayload, + }, + }); + } + + return rules; +} + +/** + * Write share grants to the host store. + */ +export async function materializeShareGrants( + store: WritableGrantStore, + input: MaterializeShareGrantsInput, +): Promise<{ written: number; grants: GrantRule[] }> { + const grants = buildShareGrants(input); + for (const grant of grants) { + await store.putGrant(grant); + } + return { written: grants.length, grants }; +} + +/** + * Split proposed access tags into those already covered by the source + * audience vs those that widen (need source-owner approval). + * + * Write-narrow-then-widen: caller writes with `allowed` only, then widens + * after approval by appending `needsApproval` tags + materializing grants. + */ +export function splitAudienceWiden( + sourceAccessTags: readonly string[], + proposedAccessTags: readonly string[], +): { allowed: string[]; needsApproval: string[] } { + const source = new Set(sourceAccessTags); + const allowed: string[] = []; + const needsApproval: string[] = []; + for (const tag of proposedAccessTags) { + if (source.has(tag)) allowed.push(tag); + else needsApproval.push(tag); + } + return { allowed, needsApproval }; +} + +/** Receipt shape stored on version attributes after widen approval. */ +export type ShareWidenReceipt = { + approvedBy: string; + approvedAt: string; + tags: string[]; + sourceVersionId: string; +}; + +export function shareWidenReceipt(params: { + approvedBy: string; + tags: readonly string[]; + sourceVersionId: string; + approvedAt?: Date; +}): ShareWidenReceipt { + return { + approvedBy: params.approvedBy, + approvedAt: (params.approvedAt ?? new Date()).toISOString(), + tags: [...params.tags], + sourceVersionId: params.sourceVersionId, + }; +} diff --git a/src/services/timeline.test.ts b/src/services/timeline.test.ts index b623560..5967c6d 100644 --- a/src/services/timeline.test.ts +++ b/src/services/timeline.test.ts @@ -3,6 +3,7 @@ import { createInMemoryGrantStore } from "@intx/authz"; import { PgDialect } from "drizzle-orm/pg-core"; import { + activeTimelineVersionJoin, filterTimelineRows, timelineWhere, type TimelineRow, @@ -96,3 +97,21 @@ describe("timelineWhere", () => { expect(sql).not.toContain("visibility_principal_ids"); }); }); + +describe("activeTimelineVersionJoin", () => { + it("filters active status and live generation by default", () => { + const { sql, params } = dialect.sqlToQuery(activeTimelineVersionJoin()!); + expect(sql).toContain("status"); + expect(sql).toContain("generation"); + expect(params).toContain("active"); + expect(params).toContain("live"); + }); + + it("accepts a replay generation tag", () => { + const { params } = dialect.sqlToQuery( + activeTimelineVersionJoin("replay_run_1")!, + ); + expect(params).toContain("replay_run_1"); + expect(params).not.toContain("live"); + }); +}); diff --git a/src/services/timeline.ts b/src/services/timeline.ts index 4788203..4f68f70 100644 --- a/src/services/timeline.ts +++ b/src/services/timeline.ts @@ -10,6 +10,7 @@ import { and, desc, eq, sql } from "drizzle-orm"; import type { ConditionRegistry, GrantStore } from "@intx/authz"; import { canAccessDocument } from "../grant-tags.ts"; +import { LIVE_GENERATION } from "../core/generation.ts"; import type { Db } from "../db/client.ts"; import { memoryDocument, memoryVersion } from "../db/schema.ts"; @@ -31,6 +32,11 @@ export type ListTimelineParams = { /** Host grant store — required for non-creator document access. */ grants?: GrantStore; conditionRegistry?: ConditionRegistry; + /** + * Replay-generation tag. Defaults to live so staged replay versions never + * appear in the default timeline (matches hybrid search). + */ + generation?: string; }; export type TimelineRow = { @@ -55,6 +61,20 @@ export function timelineWhere(tenantId: string) { return eq(memoryDocument.tenantId, tenantId); } +/** + * Active-version join for timeline: status + generation (live by default). + * Exported so tests can assert the generation predicate without a live DB. + */ +export function activeTimelineVersionJoin( + generation: string = LIVE_GENERATION, +) { + return and( + eq(memoryVersion.documentId, memoryDocument.id), + eq(memoryVersion.status, "active"), + eq(memoryVersion.generation, generation), + ); +} + /** * Filter raw timeline rows to those the principal may see under grant tags. */ @@ -108,6 +128,7 @@ export async function filterTimelineRows( /** * List recent document events for a tenant, filtered by grant-tag access. + * Only active versions in the requested generation (default live) appear. */ export async function listTimelineEvents( params: ListTimelineParams, @@ -117,6 +138,7 @@ export async function listTimelineEvents( MAX_LIMIT, ); const fetchLimit = Math.min(limit * TIMELINE_OVERFETCH, MAX_LIMIT * TIMELINE_OVERFETCH); + const generation = params.generation ?? LIVE_GENERATION; const rows = await params.db .select({ @@ -129,13 +151,7 @@ export async function listTimelineEvents( accessTags: memoryDocument.accessTags, }) .from(memoryDocument) - .innerJoin( - memoryVersion, - and( - eq(memoryVersion.documentId, memoryDocument.id), - eq(memoryVersion.status, "active"), - ), - ) + .innerJoin(memoryVersion, activeTimelineVersionJoin(generation)) .where(timelineWhere(params.tenantId)) .orderBy(desc(memoryVersion.occurredAt)) .limit(fetchLimit); diff --git a/src/services/transform.test.ts b/src/services/transform.test.ts index c640faf..d0453ea 100644 --- a/src/services/transform.test.ts +++ b/src/services/transform.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from "bun:test"; import { type } from "arktype"; -import { buildRerankClientConfig } from "./transform.ts"; +import { + buildRerankClientConfig, + isPromotableRunStatus, +} from "./transform.ts"; import { TransformConfigParamsSchema } from "../core/schemas/transform.ts"; describe("buildRerankClientConfig", () => { @@ -31,6 +34,14 @@ describe("buildRerankClientConfig", () => { }); }); +describe("isPromotableRunStatus", () => { + it("allows only completed runs", () => { + expect(isPromotableRunStatus("completed")).toBe(true); + expect(isPromotableRunStatus("running")).toBe(false); + expect(isPromotableRunStatus("failed")).toBe(false); + }); +}); + describe("TransformConfigParamsSchema", () => { it("accepts a fully-specified params object", () => { const parsed = TransformConfigParamsSchema({ diff --git a/src/services/transform.ts b/src/services/transform.ts index 39fea77..af7b3c7 100644 --- a/src/services/transform.ts +++ b/src/services/transform.ts @@ -4,7 +4,11 @@ import type { Db, RawSql } from "../db/client.ts"; import type { EngineConfig } from "../config.ts"; import { newId } from "../core/id.ts"; import { formatCaughtError, log } from "../log.ts"; -import { rawCapture, transformConfig, transformRun } from "../db/schema.ts"; +import { + rawCapture, + transformConfig, + transformRun, +} from "../db/schema.ts"; import { TransformConfigParamsSchema, type TransformConfigParams, @@ -17,6 +21,9 @@ import type { Chunker } from "../core/chunk/types.ts"; import { EmbedClientConfigSchema, type EmbedClientConfig } from "../core/embed-client.ts"; import type { RerankClientConfig } from "../core/rerank-client.ts"; import { deriveFromRawCapture, type CaptureInput } from "./capture.ts"; +import { LIVE_GENERATION } from "../core/generation.ts"; +import { activateEmbedModel, activateEmbedModelByKey, clearActiveEmbedModels, resolveActiveEmbedTable } from "../core/embed-model-registry.ts"; +import { createRawSqlClient } from "../core/embed-sql.ts"; export class TransformConfigNotFoundError extends Error { constructor(configId: string) { @@ -46,6 +53,9 @@ export interface TransformRunRow { error: string | null; createdAt: Date; completedAt: Date | null; + archivedLiveGeneration: string | null; + archivedLiveModelKey: string | null; + promotedAt: Date | null; } // Parses the jsonb `params` column at this trust boundary — the row was @@ -161,6 +171,9 @@ async function loadTransformRun( error: row.error, createdAt: row.createdAt, completedAt: row.completedAt, + archivedLiveGeneration: row.archivedLiveGeneration ?? null, + archivedLiveModelKey: row.archivedLiveModelKey ?? null, + promotedAt: row.promotedAt ?? null, }; } @@ -225,21 +238,35 @@ export interface GenerationSearchParams { mmrLambda: number | undefined; overfetch: number | undefined; rerank: RerankClientConfig | undefined; + /** Fully-resolved embed client for this generation's transform_config. */ + embed: EmbedClientConfig | undefined; } // Resolves a search-time `generation` (a transform_run id, per the 1:1 // `transform_run.generation` uniqueness) back to its config's tuning knobs. -// Returns `null` when the generation isn't a known replay run (including -// 'live', which the caller should never even ask this for) — hybridSearch -// falls back to its own engine defaults for every field in that case. +// Returns `null` when the generation isn't a known replay run for this tenant +// (including 'live', which the caller should never even ask this for) — +// hybridSearch falls back to its own engine defaults for every field in that +// case. Tenant is required so a cross-tenant generation id cannot resolve +// another tenant's embed overrides (which may carry apiKey). +// +// `engineEmbed` is required to fully resolve a partial transform embed +// override (same merge rules as runTransform). export async function resolveGenerationSearchParams( db: Db, generation: string, + engineEmbed: EngineConfig["embed"] | undefined, + tenantId: string, ): Promise { const runRows = await db .select({ configId: transformRun.configId }) .from(transformRun) - .where(eq(transformRun.generation, generation)) + .where( + and( + eq(transformRun.generation, generation), + eq(transformRun.tenantId, tenantId), + ), + ) .limit(1); const run = runRows[0]; if (!run) return null; @@ -247,18 +274,28 @@ export async function resolveGenerationSearchParams( const configRows = await db .select({ params: transformConfig.params }) .from(transformConfig) - .where(eq(transformConfig.id, run.configId)) + .where( + and( + eq(transformConfig.id, run.configId), + eq(transformConfig.tenantId, tenantId), + ), + ) .limit(1); const configRow = configRows[0]; if (!configRow) return null; const params = parseConfigParams(configRow.params); + const embed = + engineEmbed !== undefined + ? buildEmbedClientConfig(params.embed, engineEmbed) + : undefined; return { authorityWeight: params.authorityWeight, recencyHalfLifeDays: params.recencyHalfLifeDays, mmrLambda: params.mmrLambda, overfetch: params.overfetch, rerank: buildRerankClientConfig(params.rerank), + embed, }; } @@ -442,3 +479,191 @@ export async function runTransform( return loadTransformRun(deps.db, runId); } + +export class TransformPromoteError extends Error { + constructor(message: string) { + super(message); + this.name = "TransformPromoteError"; + } +} + +/** + * True when a transform_run may be promoted to live. Exported for unit tests. + * Only completed runs are promotable — failed/running leave a partial corpus. + */ +export function isPromotableRunStatus( + status: TransformRunRow["status"], +): status is "completed" { + return status === "completed"; +} + +/** + * Promote a completed staged generation to live. + * + * 1. Snapshot the current active embed model_key (for demote restore). + * 2. Activate the run's embed model first so a failed activate leaves versions + * untouched (brief dense mismatch window is preferred over committed corpus + * with no matching dense table). + * 3. Swap generations: live → archive tag, staged → live. + * 4. Record archive tag + prior model_key + promoted_at for demote. + * + * Does not delete versions. Demote reverses the generation swap and re-activates + * the prior live embed model when recorded. + */ +export async function promoteGeneration( + deps: { db: Db; sql: RawSql; config: EngineConfig }, + input: { tenantId: string; generation: string }, +): Promise { + if (input.generation === LIVE_GENERATION) { + throw new TransformPromoteError("cannot promote the live generation onto itself"); + } + + const runRows = await deps.db + .select() + .from(transformRun) + .where( + and( + eq(transformRun.generation, input.generation), + eq(transformRun.tenantId, input.tenantId), + ), + ) + .limit(1); + const run = runRows[0]; + if (!run) { + throw new TransformPromoteError( + `no transform_run for generation ${input.generation}`, + ); + } + if (run.promotedAt) { + throw new TransformPromoteError( + `generation ${input.generation} is already promoted`, + ); + } + if (!isPromotableRunStatus(run.status as TransformRunRow["status"])) { + throw new TransformPromoteError( + `generation ${input.generation} is not completed (status=${run.status})`, + ); + } + + const configRow = await loadTransformConfig(deps.db, run.configId); + if (configRow.tenantId !== input.tenantId) { + throw new TransformPromoteError( + `transform_config tenant mismatch for generation ${input.generation}`, + ); + } + const embed = buildEmbedClientConfig(configRow.params.embed, deps.config.embed); + const archiveGen = `archive_${run.id}_${Date.now()}`; + const readClient = createRawSqlClient(deps.sql); + + // Snapshot prior active model before we flip dense search. + const priorActive = await resolveActiveEmbedTable(readClient, input.tenantId); + const priorModelKey = priorActive?.modelKey ?? null; + + // Activate the staged embed model and swap generation tags inside one + // Postgres transaction. resolveActiveEmbedTable and the live-generation + // filter used by search are otherwise readable independently, which opens + // a window where dense search resolves the newly-active (staged) table + // while `version` rows are still tagged with the pre-swap generation — + // a silent, empty-result degradation of live dense search. Doing both + // writes in one transaction means any concurrent reader sees only the + // fully-pre-promote or fully-post-promote state, never the half-way one. + // A thrown error here rolls back the embed activation too, so no separate + // restore-on-failure step is needed. + await deps.sql.begin(async (txSql) => { + const txClient = createRawSqlClient(txSql); + await activateEmbedModel(txClient, input.tenantId, embed); + + // 1) archive current live + await txSql.unsafe( + `UPDATE "memory"."version" SET generation = $1 WHERE tenant_id = $2 AND generation = $3`, + [archiveGen, input.tenantId, LIVE_GENERATION], + ); + // 2) promote staged → live + await txSql.unsafe( + `UPDATE "memory"."version" SET generation = $1 WHERE tenant_id = $2 AND generation = $3`, + [LIVE_GENERATION, input.tenantId, input.generation], + ); + // 3) bookkeeping — generation column stays the original run id for lookup; + // versions now live under 'live'. Search by generation=runId after + // promote finds nothing (expected); demote restores. + await txSql.unsafe( + `UPDATE "memory"."transform_run" SET archived_live_generation = $1, archived_live_model_key = $2, promoted_at = now() WHERE id = $3`, + [archiveGen, priorModelKey, run.id], + ); + }); + + return loadTransformRun(deps.db, run.id); +} + +/** + * Demote a previously promoted generation: swap archive back to live and + * move the demoted live corpus back onto the run's generation tag. Restores + * the pre-promote active embed model when one was recorded. + */ +export async function demoteGeneration( + deps: { db: Db; sql: RawSql; config: EngineConfig }, + input: { tenantId: string; generation: string }, +): Promise { + const runRows = await deps.db + .select() + .from(transformRun) + .where( + and( + eq(transformRun.generation, input.generation), + eq(transformRun.tenantId, input.tenantId), + ), + ) + .limit(1); + const run = runRows[0]; + if (!run) { + throw new TransformPromoteError( + `no transform_run for generation ${input.generation}`, + ); + } + if (!run.promotedAt || !run.archivedLiveGeneration) { + throw new TransformPromoteError( + `generation ${input.generation} is not currently promoted`, + ); + } + + const archiveGen = run.archivedLiveGeneration; + const priorModelKey = run.archivedLiveModelKey ?? null; + + // Restore prior dense target and rewrite generation tags inside one + // Postgres transaction — see promoteGeneration for why: doing these as + // separate writes lets a concurrent reader observe the restored model + // paired with not-yet-swapped generation tags (or vice versa), silently + // emptying live dense search for the duration of the gap. A thrown error + // here rolls back the restore too, so demote is all-or-nothing. + await deps.sql.begin(async (txSql) => { + const txClient = createRawSqlClient(txSql); + if (priorModelKey) { + try { + await activateEmbedModelByKey(txClient, input.tenantId, priorModelKey); + } catch (err) { + throw new TransformPromoteError( + `cannot demote: failed to restore prior embed model ${priorModelKey}: ${formatCaughtError(err)}`, + ); + } + } else { + await clearActiveEmbedModels(txClient, input.tenantId); + } + + // live (promoted) → back to run generation + await txSql.unsafe( + `UPDATE "memory"."version" SET generation = $1 WHERE tenant_id = $2 AND generation = $3`, + [input.generation, input.tenantId, LIVE_GENERATION], + ); + // archive → live + await txSql.unsafe( + `UPDATE "memory"."version" SET generation = $1 WHERE tenant_id = $2 AND generation = $3`, + [LIVE_GENERATION, input.tenantId, archiveGen], + ); + await txSql.unsafe( + `UPDATE "memory"."transform_run" SET archived_live_generation = NULL, archived_live_model_key = NULL, promoted_at = NULL WHERE id = $1`, + [run.id], + ); + }); + + return loadTransformRun(deps.db, run.id); +} diff --git a/src/tools/add.ts b/src/tools/add.ts index 08e4373..82b4c8e 100644 --- a/src/tools/add.ts +++ b/src/tools/add.ts @@ -12,6 +12,30 @@ function parseAddArgs(args: Record): MemoryAddBody { if (parsed.access_tags !== undefined) { body.access_tags = parsed.access_tags; } + if (parsed.kind !== undefined) { + body.kind = parsed.kind; + } + if (parsed.generator_agent_id !== undefined) { + body.generator_agent_id = parsed.generator_agent_id; + } + if (parsed.provenance !== undefined) { + body.provenance = parsed.provenance; + } + if (parsed.lineage_class !== undefined) { + body.lineage_class = parsed.lineage_class; + } + if (parsed.temporal_class !== undefined) { + body.temporal_class = parsed.temporal_class; + } + if (parsed.derived_from !== undefined) { + body.derived_from = parsed.derived_from; + } + if (parsed.valid_from !== undefined) { + body.valid_from = parsed.valid_from; + } + if (parsed.valid_until !== undefined) { + body.valid_until = parsed.valid_until; + } if (parsed.share !== undefined) { // Rebuild share field-by-field — arktype keeps undeclared nested keys. const share: NonNullable = {}; @@ -33,15 +57,17 @@ function parseAddArgs(args: Record): MemoryAddBody { * Installable tool: POST /api/tenants/:tenantId/memory/add. * * Tenant and auth come from env (`memoryTenantId`, `memoryAuthToken`); - * model args never carry identity. + * model args never carry identity. Distiller claims pass + * `generator_agent_id` + `derived_from` for loop-safety and lineage. */ export const memoryAdd = defineMemoryHttpTool({ id: "@corbits/memory/add", name: "memory_add", description: - "Store a note in tenant memory. Returns { documentId }. " + - "Identity is the authenticated principal on the hub; do not " + - "pass tenant or principal ids.", + "Store a note in tenant memory. Returns { documentId, versionId }. " + + "For distilled claims set generator_agent_id, provenance=inferred, " + + "lineage_class=derived, and derived_from source version ids. " + + "Identity is the authenticated principal on the hub.", inputSchema: { type: "object", properties: { @@ -57,7 +83,46 @@ export const memoryAdd = defineMemoryHttpTool({ type: "array", items: { type: "string" }, description: - "Optional grant-pattern tags controlling document visibility", + "Optional grant-pattern tags controlling document visibility " + + "(distiller: copy from source feed entry, never widen)", + }, + kind: { + type: "string", + description: "Document kind (default note)", + }, + generator_agent_id: { + type: "string", + description: + "Agent id that authored this version (e.g. resident-distiller). " + + "Enables feed excludeGenerator loop-safety.", + }, + provenance: { + type: "string", + enum: ["stated", "inferred", "unknown"], + description: "How content was obtained (inferred for distilled claims)", + }, + lineage_class: { + type: "string", + enum: ["native", "imported", "derived"], + description: "Data lineage (derived for distilled claims)", + }, + temporal_class: { + type: "string", + enum: ["event", "deadline", "state", "lesson"], + description: "Temporal ranking class", + }, + derived_from: { + type: "array", + items: { type: "string" }, + description: "Source version ids this claim is derived from", + }, + valid_from: { + type: "string", + description: "Optional validity start (ISO)", + }, + valid_until: { + type: "string", + description: "Optional validity end (ISO)", }, share: { type: "object", diff --git a/src/tools/client.ts b/src/tools/client.ts index 37cef5e..02b44e1 100644 --- a/src/tools/client.ts +++ b/src/tools/client.ts @@ -26,6 +26,14 @@ export type MemoryHttpClient = { add(body: MemoryAddBody, signal?: AbortSignal): Promise; search(body: MemorySearchBody, signal?: AbortSignal): Promise; list(limit?: number, signal?: AbortSignal): Promise; + feed( + opts?: { + after?: number; + limit?: number; + excludeGenerator?: string; + }, + signal?: AbortSignal, + ): Promise; }; /** Cap hub error text embedded in tool errors (avoid huge/secret-ish dumps). */ @@ -127,6 +135,23 @@ export function createMemoryHttpClient( ...(signal !== undefined ? { signal } : {}), }); }, + feed(opts, signal) { + const params = new URLSearchParams(); + if (opts?.after !== undefined) { + params.set("after", String(opts.after)); + } + if (opts?.limit !== undefined) { + params.set("limit", String(opts.limit)); + } + if (opts?.excludeGenerator !== undefined) { + params.set("exclude_generator", opts.excludeGenerator); + } + const qs = params.toString(); + return request(`/feed${qs ? `?${qs}` : ""}`, { + method: "GET", + ...(signal !== undefined ? { signal } : {}), + }); + }, }; } diff --git a/src/tools/feed.ts b/src/tools/feed.ts new file mode 100644 index 0000000..795b8e4 --- /dev/null +++ b/src/tools/feed.ts @@ -0,0 +1,78 @@ +import { type } from "arktype"; + +import { parseWithArk } from "../http-bodies.ts"; +import { defineMemoryHttpTool } from "./install.ts"; + +const FeedArgs = type({ + "after?": "number.integer >= 0", + "limit?": "1 <= number.integer <= 100", + "exclude_generator?": "string", +}); + +function coerceFeedArgs(args: Record): Record { + const out = { ...args }; + for (const key of ["after", "limit"] as const) { + const raw = out[key]; + if (typeof raw === "string" && raw.trim() !== "") { + const n = Number(raw); + if (Number.isFinite(n)) out[key] = n; + } + } + return out; +} + +/** + * Installable tool: GET /api/tenants/:tenantId/memory/feed. + * + * Distiller pull surface — always pass exclude_generator matching the + * writer's generator_agent_id so the agent never re-consumes its own claims. + */ +export const memoryFeed = defineMemoryHttpTool({ + id: "@corbits/memory/feed", + name: "memory_feed", + description: + "Pull new memory versions after a cursor (capture feed). " + + "Returns { entries, nextCursor }. Always set exclude_generator to your " + + "generator_agent_id (e.g. resident-distiller) for loop-safety. " + + "Copy accessTags from each entry onto distilled writes — never widen.", + inputSchema: { + type: "object", + properties: { + after: { + type: "integer", + minimum: 0, + description: "Exclusive cursor (feed_seq > after). Default 0.", + }, + limit: { + type: "integer", + minimum: 1, + maximum: 100, + description: "Page size (1–100, default 50)", + }, + exclude_generator: { + type: "string", + description: + "Skip versions written by this generator_agent_id (loop-safety)", + }, + }, + additionalProperties: false, + }, + async handle(client, args, signal) { + const parsed = parseWithArk( + FeedArgs, + coerceFeedArgs(args), + "memory_feed", + ); + const result = await client.feed( + { + ...(parsed.after !== undefined ? { after: parsed.after } : {}), + ...(parsed.limit !== undefined ? { limit: parsed.limit } : {}), + ...(parsed.exclude_generator !== undefined + ? { excludeGenerator: parsed.exclude_generator } + : {}), + }, + signal, + ); + return JSON.stringify(result); + }, +}); diff --git a/src/tools/index.ts b/src/tools/index.ts index 0249836..95d31fc 100644 --- a/src/tools/index.ts +++ b/src/tools/index.ts @@ -9,6 +9,7 @@ export { memoryAdd } from "./add.ts"; export { memorySearch } from "./search.ts"; export { memoryList } from "./list.ts"; +export { memoryFeed } from "./feed.ts"; export { createMemoryHttpClient, MEMORY_TOOL_ENV_KEYS, @@ -16,3 +17,13 @@ export { type MemoryHttpConfig, type MemoryToolEnv, } from "./client.ts"; + +/** Re-export installer grant requirements (same as package root). */ +export { + MEMORY_CAPABILITY_IDS, + MEMORY_GRANT_REQUIREMENTS, + type MemoryGrantRequirement, + type MemoryGrantSource, + type MemoryGrantSurface, +} from "../grant-requirements.ts"; + diff --git a/src/tools/search.ts b/src/tools/search.ts index 77dd9b8..6814517 100644 --- a/src/tools/search.ts +++ b/src/tools/search.ts @@ -21,6 +21,9 @@ function parseSearchArgs(args: Record): MemorySearchBody { if (parsed.includeEvidence !== undefined) { body.includeEvidence = parsed.includeEvidence; } + if (parsed.includeDeprecated !== undefined) { + body.includeDeprecated = parsed.includeDeprecated; + } return body; } @@ -34,8 +37,10 @@ export const memorySearch = defineMemoryHttpTool({ name: "memory_search", description: "Hybrid semantic + keyword search over tenant memory. " + - "Returns ranked items (and optional evidence). Identity is " + - "the authenticated principal on the hub.", + "Returns ranked items with optional additive attribution " + + "(provenance, temporal class, corroboration, derivedFrom) and evidence. " + + "Attribute stated content to the actor; treat inferred as own-voice claims. " + + "Identity is the authenticated principal on the hub.", inputSchema: { type: "object", properties: { @@ -70,6 +75,11 @@ export const memorySearch = defineMemoryHttpTool({ description: "Include evidence strength on the response (hub default true)", }, + includeDeprecated: { + type: "boolean", + description: + "Include deprecated versions in results (default false)", + }, }, required: ["query"], additionalProperties: false, diff --git a/src/tools/tools.test.ts b/src/tools/tools.test.ts index 09f7fd5..75bc4e1 100644 --- a/src/tools/tools.test.ts +++ b/src/tools/tools.test.ts @@ -115,7 +115,7 @@ describe("createMemoryHttpClient", () => { test("POSTs add under tenant path with Bearer auth", async () => { const { calls, fetchMock } = makeFetchMock(() => ({ status: 200, - json: { documentId: "doc-1" }, + json: { documentId: "doc-1", versionId: "ver-1" }, })); const client = createMemoryHttpClient({ baseUrl: `${BASE}///`, @@ -124,7 +124,7 @@ describe("createMemoryHttpClient", () => { fetch: fetchMock, }); const out = await client.add({ title: "t", text: "body" }); - expect(out).toEqual({ documentId: "doc-1" }); + expect(out).toEqual({ documentId: "doc-1", versionId: "ver-1" }); expect(calls).toHaveLength(1); const c = calls[0]!; expect(c.method).toBe("POST"); @@ -233,7 +233,7 @@ describe("memoryAdd factory", () => { test("happy path: body has no identity fields", async () => { const { calls, fetchMock } = makeFetchMock(() => ({ status: 200, - json: { documentId: "doc-9" }, + json: { documentId: "doc-9", versionId: "ver-9" }, })); const bundle = memoryAdd(toolEnv({ memoryFetch: fetchMock })); expect(bundle.definitions.map((d) => d.name)).toEqual(["memory_add"]); @@ -246,7 +246,7 @@ describe("memoryAdd factory", () => { new AbortController().signal, ); expect(result.isError).toBeFalsy(); - expect(result.content).toBe(JSON.stringify({ documentId: "doc-9" })); + expect(result.content).toBe(JSON.stringify({ documentId: "doc-9", versionId: "ver-9" })); const body = JSON.parse(calls[0]!.body ?? "{}") as Record; expect(body).not.toHaveProperty("tenantId"); expect(body).not.toHaveProperty("principalId");