From a62081765479784d3e6bbbe3b639fe94799e15a2 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 10 Aug 2026 02:21:32 -0700 Subject: [PATCH 01/19] Make versions claim-bearing with provenance and locked edge enums Align arktype edge/lineage enums with the Postgres CHECK set, split ranking sourceClass from stored lineage, and add version.provenance so derived claims can be written with derived_from edges without putting inference in core. Capture resolves native edge endpoints to principal entities and no longer writes ranking priors into the lineage column (which would fail the CHECK). --- IMPLEMENTATION.md | 24 ++++- migrations/0003_claim_bearing.sql | 13 +++ src/core/adapt-and-plan.test.ts | 4 +- src/core/enums.lockstep.test.ts | 114 ++++++++++++++++++++++ src/core/enums.ts | 58 +++++++++++ src/core/schemas/adapted-document.test.ts | 2 +- src/core/schemas/adapted-document.ts | 16 ++- src/core/schemas/claim-bearing.test.ts | 93 ++++++++++++++++++ src/core/schemas/document.test.ts | 4 + src/core/schemas/document.ts | 17 ++++ src/core/schemas/entity-edge.test.ts | 14 ++- src/core/schemas/entity-edge.ts | 16 ++- src/db/schema.ts | 3 + src/services/capture.ts | 52 ++++++++-- 14 files changed, 409 insertions(+), 21 deletions(-) create mode 100644 migrations/0003_claim_bearing.sql create mode 100644 src/core/enums.lockstep.test.ts create mode 100644 src/core/enums.ts create mode 100644 src/core/schemas/claim-bearing.test.ts diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md index 49143a3..05a45ce 100644 --- a/IMPLEMENTATION.md +++ b/IMPLEMENTATION.md @@ -196,9 +196,27 @@ Lightweight graph rows. `memory_entity` has no unique constraint; dedupe on `(tenant_id, kind, identifiers)` is done in application code (`upsertEntity` in `capture.ts`, an exact-match linear scan per kind). Same for `memory_edge` (dedupe on the full `(tenant_id, rel, from, to)` tuple, -`upsertEdge`). `rel` is constrained (DB CHECK + arktype) to -`'about'|'produced_by'|'links'|'parent'|'mentions'|'waiting_on'`; `from_type`/ -`to_type` to `'document'|'entity'|'native'`. +`upsertEdge`). `rel` is constrained (DB CHECK + arktype, single source of +truth in `src/core/enums.ts`) to +`mentions|about|authored_by|involves|part_of|derived_from|supports|contradicts|supersedes`; +`from_type`/`to_type` stored in the DB are +`document|version|chunk|entity`. Adapter-facing edge hints may also use +`native` as a planning-time principal ref; capture resolves it to an +`entity` row (`kind=principal`) before insert. A lockstep test asserts the +migration CHECK sets match the TS constants. + +### Provenance and lineage (claim-bearing substrate) + +Two axes on `memory.version`, orthogonal to ranking priors: + +| Column / field | Values | Meaning | +| --- | --- | --- | +| `provenance` | `stated` \| `inferred` \| `unknown` | How content was obtained. Capture defaults to `stated`; distilled claims write `inferred`. Existing rows default `unknown`. | +| `source_class` (lineage) | `native` \| `imported` \| `derived` | Data lineage. Adapters write `native` (or `imported` for bulk import); distilled claims write `derived` via `AdaptedDocument.lineageClass`. | + +Ranking priors (`AdaptedDocument.sourceClass`: `native|thread|channel|call|record`) feed `computeAuthority` only and are **not** written to the version `source_class` column — that was a latent CHECK violation fixed when the axes were split. + +A derived claim is a normal version with `provenance: inferred`, `lineageClass: derived`, and a `derived_from` edge to the source version (or document). Core never runs inference; it only accepts the shape. ### `memory_embed_model` Per-tenant registry of which embed model is currently active, and the diff --git a/migrations/0003_claim_bearing.sql b/migrations/0003_claim_bearing.sql new file mode 100644 index 0000000..94e38a4 --- /dev/null +++ b/migrations/0003_claim_bearing.sql @@ -0,0 +1,13 @@ +-- Claim-bearing substrate: provenance mode on version. +-- Edge rel / ref-type CHECKs already match the canonical set in 0002; +-- this migration only adds how-content-was-obtained (stated vs inferred). + +ALTER TABLE "memory"."version" + ADD COLUMN IF NOT EXISTS "provenance" text NOT NULL DEFAULT 'unknown'; + +ALTER TABLE "memory"."version" + DROP CONSTRAINT IF EXISTS "version_provenance_check"; + +ALTER TABLE "memory"."version" + ADD CONSTRAINT "version_provenance_check" + CHECK ("provenance" IN ('stated', 'inferred', 'unknown')); diff --git a/src/core/adapt-and-plan.test.ts b/src/core/adapt-and-plan.test.ts index 56def50..855b111 100644 --- a/src/core/adapt-and-plan.test.ts +++ b/src/core/adapt-and-plan.test.ts @@ -51,12 +51,12 @@ describe("adaptAndPlan", () => { it("carries entityHints and edges through from the adapted document", () => { const plan = adaptAndPlan( validAdaptedDocument({ - edges: [{ rel: "links", to: { type: "native", ref: "mail:m1" } }], + edges: [{ rel: "involves", to: { type: "native", ref: "mail:m1" } }], entityHints: [{ kind: "person", identifier: "jane@example.com" }], }), ); expect(plan.edges).toEqual([ - { rel: "links", to: { type: "native", ref: "mail:m1" } }, + { rel: "involves", to: { type: "native", ref: "mail:m1" } }, ]); expect(plan.entityHints).toEqual([ { kind: "person", identifier: "jane@example.com" }, diff --git a/src/core/enums.lockstep.test.ts b/src/core/enums.lockstep.test.ts new file mode 100644 index 0000000..b56eef3 --- /dev/null +++ b/src/core/enums.lockstep.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from "bun:test"; +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { + EDGE_RELS, + EDGE_REF_TYPES_DB, + LINEAGE_CLASSES, + PROVENANCE_MODES, +} from "./enums.ts"; +import { MemoryEdgeRelSchema, MemoryEdgeRefTypeSchema } from "./schemas/entity-edge.ts"; +import { + LineageClassSchema, + ProvenanceModeSchema, +} from "./schemas/document.ts"; +import { type } from "arktype"; + +const MIGRATIONS_DIR = join(import.meta.dir, "../../migrations"); + +function allMigrationSql(): string { + const files = readdirSync(MIGRATIONS_DIR) + .filter((f) => f.endsWith(".sql")) + .sort(); + return files + .map((f) => readFileSync(join(MIGRATIONS_DIR, f), "utf8")) + .join("\n"); +} + +/** Pull the last CHECK (... IN (...)) body for a named constraint. */ +function lastCheckInList(sql: string, constraintName: string): string[] { + const re = new RegExp( + `CONSTRAINT\\s+"${constraintName}"\\s+CHECK\\s*\\(\\s*"[^"]+"\\s+IN\\s*\\(([\\s\\S]*?)\\)\\s*\\)`, + "gi", + ); + let match: RegExpExecArray | null; + let last: string | null = null; + while ((match = re.exec(sql)) !== null) { + last = match[1] ?? null; + } + if (last === null) { + throw new Error(`constraint ${constraintName} not found in migrations`); + } + return [...last.matchAll(/'([^']+)'/g)].map((m) => m[1] as string); +} + +function sorted(values: readonly string[]): string[] { + return [...values].sort(); +} + +describe("enum lockstep: TS constants match migration CHECK constraints", () => { + const sql = allMigrationSql(); + + it("edge_rel_check matches EDGE_RELS", () => { + expect(sorted(lastCheckInList(sql, "edge_rel_check"))).toEqual( + sorted(EDGE_RELS), + ); + }); + + it("edge_from_type_check matches EDGE_REF_TYPES_DB", () => { + expect(sorted(lastCheckInList(sql, "edge_from_type_check"))).toEqual( + sorted(EDGE_REF_TYPES_DB), + ); + }); + + it("edge_to_type_check matches EDGE_REF_TYPES_DB", () => { + expect(sorted(lastCheckInList(sql, "edge_to_type_check"))).toEqual( + sorted(EDGE_REF_TYPES_DB), + ); + }); + + it("version_source_class_check matches LINEAGE_CLASSES", () => { + expect(sorted(lastCheckInList(sql, "version_source_class_check"))).toEqual( + sorted(LINEAGE_CLASSES), + ); + }); + + it("version_provenance_check matches PROVENANCE_MODES", () => { + expect(sorted(lastCheckInList(sql, "version_provenance_check"))).toEqual( + sorted(PROVENANCE_MODES), + ); + }); +}); + +describe("enum lockstep: arktype accepts every SSOT value and rejects unknown", () => { + it("MemoryEdgeRelSchema accepts all EDGE_RELS", () => { + for (const rel of EDGE_RELS) { + const out = MemoryEdgeRelSchema(rel); + expect(out instanceof type.errors ? out.summary : out).toBe(rel); + } + expect(MemoryEdgeRelSchema("produced_by") instanceof type.errors).toBe( + true, + ); + }); + + it("MemoryEdgeRefTypeSchema accepts adapter set including native", () => { + for (const t of ["document", "version", "chunk", "entity", "native"] as const) { + const out = MemoryEdgeRefTypeSchema(t); + expect(out instanceof type.errors ? out.summary : out).toBe(t); + } + }); + + it("LineageClassSchema accepts LINEAGE_CLASSES only", () => { + for (const c of LINEAGE_CLASSES) { + expect(LineageClassSchema(c) instanceof type.errors).toBe(false); + } + expect(LineageClassSchema("thread") instanceof type.errors).toBe(true); + }); + + it("ProvenanceModeSchema accepts PROVENANCE_MODES only", () => { + for (const p of PROVENANCE_MODES) { + expect(ProvenanceModeSchema(p) instanceof type.errors).toBe(false); + } + expect(ProvenanceModeSchema("guessed") instanceof type.errors).toBe(true); + }); +}); diff --git a/src/core/enums.ts b/src/core/enums.ts new file mode 100644 index 0000000..9daae5b --- /dev/null +++ b/src/core/enums.ts @@ -0,0 +1,58 @@ +// Single source of truth for memory-plane enums that also appear as +// Postgres CHECK constraints. Arktype schemas import these; the lockstep +// test asserts the latest migration SQL matches exactly. + +/** Graph edge relationship kinds stored on memory.edge.rel. */ +export const EDGE_RELS = [ + "mentions", + "about", + "authored_by", + "involves", + "part_of", + "derived_from", + "supports", + "contradicts", + "supersedes", +] as const; +export type EdgeRel = (typeof EDGE_RELS)[number]; + +/** + * Endpoint kinds that may be written to memory.edge.from_type / to_type. + * Adapter-facing hints may also use `native` (see EDGE_REF_TYPES_ADAPTER); + * capture resolves native → entity before insert. + */ +export const EDGE_REF_TYPES_DB = [ + "document", + "version", + "chunk", + "entity", +] as const; +export type EdgeRefTypeDb = (typeof EDGE_REF_TYPES_DB)[number]; + +/** Adapter-facing edge endpoint kinds, including planning-time `native`. */ +export const EDGE_REF_TYPES_ADAPTER = [ + ...EDGE_REF_TYPES_DB, + "native", +] as const; +export type EdgeRefTypeAdapter = (typeof EDGE_REF_TYPES_ADAPTER)[number]; + +/** + * Data-lineage class stored on memory.version.source_class. + * Orthogonal to AuthoritySourceClass (ranking priors: thread/channel/…). + */ +export const LINEAGE_CLASSES = ["native", "imported", "derived"] as const; +export type LineageClass = (typeof LINEAGE_CLASSES)[number]; + +/** + * How the version's content was obtained relative to assertion. + * Orthogonal to created_by_kind (who) and lineageClass (where from). + */ +export const PROVENANCE_MODES = ["stated", "inferred", "unknown"] as const; +export type ProvenanceMode = (typeof PROVENANCE_MODES)[number]; + +/** Build an arktype union string from a const string array. */ +export function arktypeStringUnion( + values: readonly string[], +): string { + return values.map((v) => `'${v}'`).join("|"); +} diff --git a/src/core/schemas/adapted-document.test.ts b/src/core/schemas/adapted-document.test.ts index 7c75b01..d059c92 100644 --- a/src/core/schemas/adapted-document.test.ts +++ b/src/core/schemas/adapted-document.test.ts @@ -15,7 +15,7 @@ describe("AdaptedDocumentSchema", () => { entityHints: [{ kind: "person", identifier: "jane@example.com" }], edges: [ { rel: "about", to: { type: "entity", ref: "acme-co" } }, - { rel: "produced_by", to: { type: "native", ref: "principal_1" } }, + { rel: "authored_by", to: { type: "native", ref: "principal_1" } }, ], chunks: [ { ordinal: 0, text: "Opening remarks.", role: "summary" }, diff --git a/src/core/schemas/adapted-document.ts b/src/core/schemas/adapted-document.ts index 2dd5d3c..5f7fce7 100644 --- a/src/core/schemas/adapted-document.ts +++ b/src/core/schemas/adapted-document.ts @@ -1,5 +1,9 @@ import { type } from "arktype"; -import { CreatedByKindSchema } from "./document.ts"; +import { + CreatedByKindSchema, + LineageClassSchema, + ProvenanceModeSchema, +} from "./document.ts"; import { MemoryEdgeHintSchema } from "./entity-edge.ts"; import { AuthoritySourceClassSchema } from "../authority.ts"; @@ -48,6 +52,14 @@ export type RawPointer = typeof RawPointerSchema.infer; // // Document access is grant tags only (`accessTags`) — the security boundary // (docs/AUTHZ-DOCUMENT-ACCESS.md). +// +// Two orthogonal "class" axes on the write path: +// - sourceClass: ranking prior (thread/channel/call/record/native) for +// computeAuthority — never written to knowledge.version.source_class. +// - lineageClass: data-lineage stored on knowledge.version.source_class +// (native|imported|derived). Defaults to native at capture. +// provenance: how the content was obtained (stated|inferred|unknown); +// defaults to stated at capture for human/adapter paths. export const AdaptedDocumentSchema = type({ kind: `1 <= string <= ${MAX_KIND_CHARS}`, title: `1 <= string <= ${MAX_TITLE_CHARS}`, @@ -67,6 +79,8 @@ export const AdaptedDocumentSchema = type({ "actorCount?": "number", "sourceClass?": AuthoritySourceClassSchema, "hasSocialSignal?": "boolean", + "lineageClass?": LineageClassSchema, + "provenance?": ProvenanceModeSchema, contentHash: "string", }); export type AdaptedDocument = typeof AdaptedDocumentSchema.infer; diff --git a/src/core/schemas/claim-bearing.test.ts b/src/core/schemas/claim-bearing.test.ts new file mode 100644 index 0000000..3c02743 --- /dev/null +++ b/src/core/schemas/claim-bearing.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from "bun:test"; +import { type } from "arktype"; +import { AdaptedDocumentSchema } from "./adapted-document.ts"; +import { MemoryEdgeHintSchema } from "./entity-edge.ts"; +import type { AdaptedDocument } from "./adapted-document.ts"; + +/** + * Claim-bearing write shape: a distilled claim is a normal AdaptedDocument + * with inferred provenance, derived lineage, and a derived_from edge. + * Core never runs inference — it only accepts this shape. + */ +describe("claim-bearing AdaptedDocument shape", () => { + it("accepts a derived claim with provenance, lineage, and derived_from", () => { + const claim: AdaptedDocument = { + kind: "claim", + title: "Acme renews in Q3", + externalRef: "claim:acme-q3-renewal", + accessTags: ["memory.tenant:t1"], + entityHints: [{ kind: "org", identifier: "acme.com" }], + edges: [ + { + rel: "derived_from", + to: { type: "version", ref: "kver_source_1" }, + }, + { + rel: "supports", + to: { type: "version", ref: "kver_prior_claim" }, + }, + ], + chunks: [ + { + ordinal: 0, + text: "Acme is expected to renew in Q3 at a 12% expansion.", + }, + ], + actor: { kind: "agent", agentId: "distiller-v1" }, + sourceClass: "record", + lineageClass: "derived", + provenance: "inferred", + contentHash: "sha256:claim-1", + }; + const out = AdaptedDocumentSchema(claim); + expect(out instanceof type.errors ? out.summary : out).toEqual(claim); + }); + + it("accepts every claim-bearing edge rel", () => { + for (const rel of [ + "derived_from", + "supports", + "contradicts", + "supersedes", + "authored_by", + "involves", + "part_of", + ] as const) { + const out = MemoryEdgeHintSchema({ + rel, + to: { type: "version", ref: "kver_1" }, + }); + expect(out instanceof type.errors).toBe(false); + } + }); + + it("rejects pre-claim edge rels that were never DB-valid", () => { + for (const rel of ["produced_by", "links", "parent", "waiting_on"]) { + const out = MemoryEdgeHintSchema({ + rel, + to: { type: "entity", ref: "e1" }, + }); + expect(out instanceof type.errors).toBe(true); + } + }); + + it("keeps ranking sourceClass independent of lineageClass", () => { + const out = AdaptedDocumentSchema({ + kind: "call_transcript", + title: "Call", + externalRef: "call:1", + accessTags: ["memory.tenant:t1"], + entityHints: [], + chunks: [{ ordinal: 0, text: "hi" }], + contentHash: "sha256:x", + sourceClass: "channel", + lineageClass: "native", + provenance: "stated", + }); + expect(out instanceof type.errors).toBe(false); + if (!(out instanceof type.errors)) { + expect(out.sourceClass).toBe("channel"); + expect(out.lineageClass).toBe("native"); + } + }); +}); diff --git a/src/core/schemas/document.test.ts b/src/core/schemas/document.test.ts index 4645578..fdb3bc8 100644 --- a/src/core/schemas/document.test.ts +++ b/src/core/schemas/document.test.ts @@ -57,6 +57,8 @@ describe("MemoryVersionSchema", () => { deprecated_reason: null, created_by_principal_id: "principal_1", created_by_kind: "human", + provenance: "stated", + source_class: "native", }; const out = MemoryVersionSchema(fixture); expect(out instanceof type.errors ? out.summary : out).toEqual(fixture); @@ -78,6 +80,8 @@ describe("MemoryVersionSchema", () => { deprecated_reason: null, created_by_principal_id: null, created_by_kind: "human", + provenance: "stated", + source_class: "native", }); expect(out instanceof type.errors).toBe(true); }); diff --git a/src/core/schemas/document.ts b/src/core/schemas/document.ts index 4a7725b..50827e7 100644 --- a/src/core/schemas/document.ts +++ b/src/core/schemas/document.ts @@ -1,4 +1,9 @@ import { type } from "arktype"; +import { + LINEAGE_CLASSES, + PROVENANCE_MODES, + arktypeStringUnion, +} from "../enums.ts"; export const MemoryVersionStatusSchema = type( "'active'|'superseded'|'deprecated'|'archived'|'tombstoned'", @@ -8,6 +13,16 @@ export type MemoryVersionStatus = typeof MemoryVersionStatusSchema.infer; export const CreatedByKindSchema = type("'human'|'agent'|'system'|'adapter'"); export type CreatedByKind = typeof CreatedByKindSchema.infer; +export const LineageClassSchema = type( + arktypeStringUnion(LINEAGE_CLASSES) as "'native'|'imported'|'derived'", +); +export type LineageClass = typeof LineageClassSchema.infer; + +export const ProvenanceModeSchema = type( + arktypeStringUnion(PROVENANCE_MODES) as "'stated'|'inferred'|'unknown'", +); +export type ProvenanceMode = typeof ProvenanceModeSchema.infer; + // The stable logical row for a captured source, deduped on (tenant_id, // adapter, external_ref). Document access is grant tags only. export const MemoryDocumentSchema = type({ @@ -42,5 +57,7 @@ export const MemoryVersionSchema = type({ created_by_principal_id: "string | null", created_by_kind: CreatedByKindSchema, "generator_agent_id?": "string", + provenance: ProvenanceModeSchema, + source_class: LineageClassSchema, }); export type MemoryVersion = typeof MemoryVersionSchema.infer; diff --git a/src/core/schemas/entity-edge.test.ts b/src/core/schemas/entity-edge.test.ts index 7635b4b..439f158 100644 --- a/src/core/schemas/entity-edge.test.ts +++ b/src/core/schemas/entity-edge.test.ts @@ -52,7 +52,7 @@ describe("MemoryEdgeSchema", () => { describe("MemoryEdgeHintSchema", () => { it("parses a full fixture", () => { const out = MemoryEdgeHintSchema({ - rel: "produced_by", + rel: "authored_by", to: { type: "native", ref: "principal_1" }, }); expect(out instanceof type.errors).toBe(false); @@ -60,9 +60,19 @@ describe("MemoryEdgeHintSchema", () => { it("rejects a hint whose to is missing ref", () => { const out = MemoryEdgeHintSchema({ - rel: "produced_by", + rel: "authored_by", to: { type: "native" }, }); expect(out instanceof type.errors).toBe(true); }); + + it("accepts version and chunk endpoint types", () => { + for (const endpointType of ["version", "chunk"] as const) { + const out = MemoryEdgeHintSchema({ + rel: "derived_from", + to: { type: endpointType, ref: "id_1" }, + }); + expect(out instanceof type.errors).toBe(false); + } + }); }); diff --git a/src/core/schemas/entity-edge.ts b/src/core/schemas/entity-edge.ts index ff12c21..cd391f4 100644 --- a/src/core/schemas/entity-edge.ts +++ b/src/core/schemas/entity-edge.ts @@ -1,4 +1,9 @@ import { type } from "arktype"; +import { + EDGE_RELS, + EDGE_REF_TYPES_ADAPTER, + arktypeStringUnion, +} from "../enums.ts"; // A real-world thing (person, org, deal, ...) a document or chunk mentions. // Kept lightweight — identity keys only (email, domain, ...), not another @@ -12,11 +17,18 @@ export const MemoryEntitySchema = type({ }); export type MemoryEntity = typeof MemoryEntitySchema.infer; -export const MemoryEdgeRefTypeSchema = type("'document'|'entity'|'native'"); +// Adapter-facing endpoint kinds. `native` is a planning-time hint resolved +// to an entity row at the capture write boundary; it is never stored on +// memory.edge. +export const MemoryEdgeRefTypeSchema = type( + arktypeStringUnion(EDGE_REF_TYPES_ADAPTER) as + "'document'|'version'|'chunk'|'entity'|'native'", +); export type MemoryEdgeRefType = typeof MemoryEdgeRefTypeSchema.infer; export const MemoryEdgeRelSchema = type( - "'about'|'produced_by'|'links'|'parent'|'mentions'|'waiting_on'", + arktypeStringUnion(EDGE_RELS) as + "'mentions'|'about'|'authored_by'|'involves'|'part_of'|'derived_from'|'supports'|'contradicts'|'supersedes'", ); export type MemoryEdgeRel = typeof MemoryEdgeRelSchema.infer; diff --git a/src/db/schema.ts b/src/db/schema.ts index d69594e..40d572b 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -72,6 +72,9 @@ export const memoryVersion = memorySchema.table( actorCount: integer("actor_count").notNull().default(1), hasSocialSignal: boolean("has_social_signal").notNull().default(false), sourceClass: text("source_class").notNull().default("native"), + // How content was obtained: stated (asserted), inferred (derived claim), + // unknown (legacy / unset). Orthogonal to created_by_kind and source_class. + provenance: text("provenance").notNull().default("unknown"), rawCaptureId: text("raw_capture_id").references(() => rawCapture.id), // Replay-generation tag — 'live' for the normal /capture path; a replay tags every // version it writes with its own transform_run id instead, so a replayed diff --git a/src/services/capture.ts b/src/services/capture.ts index 19a2fb9..3a451d9 100644 --- a/src/services/capture.ts +++ b/src/services/capture.ts @@ -71,6 +71,10 @@ type CaptureTxResult = // defaults here rather than at the schema layer, so every capture (including // a future caller that forgets to set a signal) always produces a // well-formed AuthoritySignals rather than an undefined-riddled one. +// +// Ranking sourceClass (thread/channel/…) is deliberately NOT written to the +// version.source_class column — that column is data lineage +// (native|imported|derived) via lineageClass. See deriveLineageClass. function deriveAuthoritySignals(plan: CapturePlan): AuthoritySignals { return { createdByKind: plan.document.actor?.kind ?? "system", @@ -80,6 +84,14 @@ function deriveAuthoritySignals(plan: CapturePlan): AuthoritySignals { }; } +function deriveLineageClass(plan: CapturePlan): string { + return plan.document.lineageClass ?? "native"; +} + +function deriveProvenance(plan: CapturePlan): string { + return plan.document.provenance ?? "stated"; +} + async function insertVersion( tx: Tx, input: CaptureInput, @@ -111,7 +123,8 @@ async function insertVersion( authority: computeAuthority(authoritySignals), actorCount: authoritySignals.actorCount, hasSocialSignal: authoritySignals.hasSocialSignal, - sourceClass: authoritySignals.sourceClass, + sourceClass: deriveLineageClass(plan), + provenance: deriveProvenance(plan), rawCaptureId: opts.rawCaptureId, generation: opts.generation, }); @@ -195,13 +208,14 @@ async function insertOrReuseRawCapture( // No unique constraint backs memory_entity — dedupe here on an exact // (tenantId, kind, identifiers) match, matching what a caller re-emits for -// the same real-world thing across captures. +// the same real-world thing across captures. Returns the entity id (existing +// or freshly inserted) so edge resolution can point at it. async function upsertEntity( tx: Tx, tenantId: string, hint: EntityHint, now: Date, -): Promise { +): Promise { const identifiers = { value: hint.identifier }; const rows = await tx .select({ @@ -215,23 +229,29 @@ async function upsertEntity( eq(memoryEntity.kind, hint.kind), ), ); - const exists = rows.some( + const match = rows.find( (r) => JSON.stringify(r.identifiers) === JSON.stringify(identifiers), ); - if (exists) return; + if (match) return match.id; + const id = newId("kent"); await tx.insert(memoryEntity).values({ - id: newId("kent"), + id, tenantId, kind: hint.kind, identifiers, createdAt: now, updatedAt: now, }); + return id; } // No unique constraint backs memory_edge either — dedupe on the full // (tenantId, rel, from, to) tuple so re-ingesting the same document doesn't // pile up duplicate relationship rows across versions. +// +// Adapter-facing `native` endpoints are planning-time hints for principals +// (or other non-entity refs). Resolve them to a memory_entity row before +// insert so the DB CHECK (document|version|chunk|entity) is always satisfied. async function upsertEdge( tx: Tx, tenantId: string, @@ -239,6 +259,18 @@ async function upsertEdge( hint: MemoryEdgeHint, now: Date, ): Promise { + let toType = hint.to.type; + let toRef = hint.to.ref; + if (toType === "native") { + toRef = await upsertEntity( + tx, + tenantId, + { kind: "principal", identifier: toRef }, + now, + ); + toType = "entity"; + } + const rows = await tx .select({ id: memoryEdge.id }) .from(memoryEdge) @@ -248,8 +280,8 @@ async function upsertEdge( eq(memoryEdge.rel, hint.rel), eq(memoryEdge.fromType, "document"), eq(memoryEdge.fromRef, documentId), - eq(memoryEdge.toType, hint.to.type), - eq(memoryEdge.toRef, hint.to.ref), + eq(memoryEdge.toType, toType), + eq(memoryEdge.toRef, toRef), ), ) .limit(1); @@ -260,8 +292,8 @@ async function upsertEdge( rel: hint.rel, fromType: "document", fromRef: documentId, - toType: hint.to.type, - toRef: hint.to.ref, + toType, + toRef, createdAt: now, }); } From fecdbec53bc48deac30c8de289e225deb6597544 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 10 Aug 2026 02:50:01 -0700 Subject: [PATCH 02/19] feat: temporal classes and validity window for ranking (CL-5866) Add temporal_class (event/deadline/state/lesson) and valid_from/valid_until on knowledge.version; class-aware recency in hybrid search; fix timeline to filter by generation so staged replay never leaks into live views. --- IMPLEMENTATION.md | 13 +++++ docs/TEMPORAL.md | 57 ++++++++++++++++++++ migrations/0004_temporal_model.sql | 25 +++++++++ src/core/enums.lockstep.test.ts | 15 ++++++ src/core/enums.ts | 12 +++++ src/core/hybrid-search.test.ts | 79 ++++++++++++++++++++++++++++ src/core/hybrid-search.ts | 38 +++++++++++++ src/core/schemas/adapted-document.ts | 4 ++ src/core/schemas/document.test.ts | 2 + src/core/schemas/document.ts | 13 +++++ src/db/schema.ts | 4 ++ src/services/capture.ts | 15 ++++++ src/services/search.test.ts | 2 + src/services/search.ts | 23 ++++++-- src/services/timeline.test.ts | 19 +++++++ src/services/timeline.ts | 30 ++++++++--- 16 files changed, 341 insertions(+), 10 deletions(-) create mode 100644 docs/TEMPORAL.md create mode 100644 migrations/0004_temporal_model.sql diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md index 05a45ce..53496e2 100644 --- a/IMPLEMENTATION.md +++ b/IMPLEMENTATION.md @@ -218,6 +218,19 @@ Ranking priors (`AdaptedDocument.sourceClass`: `native|thread|channel|call|recor A derived claim is a normal version with `provenance: inferred`, `lineageClass: derived`, and a `derived_from` edge to the source version (or document). Core never runs inference; it only accepts the shape. +### Temporal model + +See `docs/TEMPORAL.md`. On `memory.version`: + +| Column | Role | +| --- | --- | +| `occurred_at` | Effective time the content refers to | +| `ingested_at` | When the plane learned it (no separate `asserted_at`) | +| `temporal_class` | `event` \| `deadline` \| `state` \| `lesson` — ranking prior | +| `valid_from` / `valid_until` | Optional validity window | + +Search multiplies fused scores by `temporalRecencyMultiplier` (class-aware). Timeline and search both filter `generation` (default live) so replay rows never leak into live views. + ### `memory_embed_model` Per-tenant registry of which embed model is currently active, and the dimensionality it was discovered at (`discoverModelDims`/`probeEmbedDims` — diff --git a/docs/TEMPORAL.md b/docs/TEMPORAL.md new file mode 100644 index 0000000..d024208 --- /dev/null +++ b/docs/TEMPORAL.md @@ -0,0 +1,57 @@ +# Temporal model + +How `memory.version` times and ranking classes work. Implementation lives +in capture (write), hybrid-search (rank), and search/timeline (query filters). + +## Four times (two stored, two derived) + +| Concept | Column / source | Meaning | +| --- | --- | --- | +| **Effective time** | `occurred_at` (required) | When the content *refers to*: event moment, state effective time, or when a deadline was established. Not dual-meaning — one meaning applied across classes. | +| **Ingestion / assertion** | `ingested_at` | When the memory plane learned the content (capture or distill write). There is no separate `asserted_at`. | +| **Validity start** | `valid_from` (nullable) | Optional window start for state/deadline claims. | +| **Validity end** | `valid_until` (nullable) | Optional window end. Required in practice for useful `deadline` ranking. | + +## `temporal_class` + +SSOT: `TEMPORAL_CLASSES` in `src/core/enums.ts`. Stored on **version** (not +document) so successive versions can change class. + +| Class | Recency prior | Notes | +| --- | --- | --- | +| `event` | Exponential decay from `occurred_at` (30-day half-life default) | Default for raw captures; preserves pre-model ranking. | +| `deadline` | Neutral far out; urgency ramp in a 7-day lookahead before `valid_until`; floor (0.7) after expiry | Still history-retrievable after expiry — not deleted. | +| `state` | Constant 1.0 while `status='active'` | Default for `provenance='inferred'` (distilled claims). Supersede via capture status, not recency. | +| `lesson` | Constant 1.0 | Always explicit; never a default. | + +Defaults at write: + +- Explicit `AdaptedDocument.temporalClass` wins. +- Else if `provenance === 'inferred'` → `state`. +- Else → `event`. + +## Ranking integration + +`applyBoosts` in `src/services/search.ts` multiplies the fused score by +`authorityBoostMultiplier × temporalRecencyMultiplier`. Lexical and dense +candidate queries both select `temporal_class` and `valid_until`. + +Deadline formula (module constant `DEADLINE_LOOKAHEAD_MS = 7d`): + +- `valid_until` null → 1.0 +- remaining ≤ 0 → `BOOST_MULTIPLIER_MIN` (0.7) +- remaining ≥ lookahead → 1.0 +- else ramp 1.0 → ~1.3 as the deadline approaches + +## Timeline generation filter + +Timeline joins only **active** versions in the requested **generation** +(default `live`). Replay-generation rows must not appear in the default +timeline (same rule as hybrid search). + +## Supersedes + +Unchanged: capture sets `status='superseded'` and `supersedes_version_id` +together. Temporal ranking does not write status. `state` “no decay until +superseded” means constant recency while active; superseded rows drop out of +live search/timeline via the status filter. diff --git a/migrations/0004_temporal_model.sql b/migrations/0004_temporal_model.sql new file mode 100644 index 0000000..2e7170d --- /dev/null +++ b/migrations/0004_temporal_model.sql @@ -0,0 +1,25 @@ +-- Temporal model: ranking class + validity window on version. +-- See docs/TEMPORAL.md. No asserted_at — occurred_at is effective time; +-- ingested_at is when the memory plane learned the content. + +ALTER TABLE "memory"."version" + ADD COLUMN IF NOT EXISTS "temporal_class" text NOT NULL DEFAULT 'event'; + +ALTER TABLE "memory"."version" + DROP CONSTRAINT IF EXISTS "version_temporal_class_check"; + +ALTER TABLE "memory"."version" + ADD CONSTRAINT "version_temporal_class_check" + CHECK ("temporal_class" IN ('event', 'deadline', 'state', 'lesson')); + +ALTER TABLE "memory"."version" + ADD COLUMN IF NOT EXISTS "valid_from" timestamp; + +ALTER TABLE "memory"."version" + ADD COLUMN IF NOT EXISTS "valid_until" timestamp; + +-- Distilled claims (inferred provenance) default to state ranking. +UPDATE "memory"."version" + SET "temporal_class" = 'state' + WHERE "provenance" = 'inferred' + AND "temporal_class" = 'event'; diff --git a/src/core/enums.lockstep.test.ts b/src/core/enums.lockstep.test.ts index b56eef3..9a9a5dc 100644 --- a/src/core/enums.lockstep.test.ts +++ b/src/core/enums.lockstep.test.ts @@ -6,11 +6,13 @@ import { EDGE_REF_TYPES_DB, LINEAGE_CLASSES, PROVENANCE_MODES, + TEMPORAL_CLASSES, } from "./enums.ts"; import { MemoryEdgeRelSchema, MemoryEdgeRefTypeSchema } from "./schemas/entity-edge.ts"; import { LineageClassSchema, ProvenanceModeSchema, + TemporalClassSchema, } from "./schemas/document.ts"; import { type } from "arktype"; @@ -78,6 +80,12 @@ describe("enum lockstep: TS constants match migration CHECK constraints", () => sorted(PROVENANCE_MODES), ); }); + + it("version_temporal_class_check matches TEMPORAL_CLASSES", () => { + expect(sorted(lastCheckInList(sql, "version_temporal_class_check"))).toEqual( + sorted(TEMPORAL_CLASSES), + ); + }); }); describe("enum lockstep: arktype accepts every SSOT value and rejects unknown", () => { @@ -111,4 +119,11 @@ describe("enum lockstep: arktype accepts every SSOT value and rejects unknown", } expect(ProvenanceModeSchema("guessed") instanceof type.errors).toBe(true); }); + + it("TemporalClassSchema accepts TEMPORAL_CLASSES only", () => { + for (const t of TEMPORAL_CLASSES) { + expect(TemporalClassSchema(t) instanceof type.errors).toBe(false); + } + expect(TemporalClassSchema("forecast") instanceof type.errors).toBe(true); + }); }); diff --git a/src/core/enums.ts b/src/core/enums.ts index 9daae5b..b7b6ff8 100644 --- a/src/core/enums.ts +++ b/src/core/enums.ts @@ -50,6 +50,18 @@ export type LineageClass = (typeof LINEAGE_CLASSES)[number]; export const PROVENANCE_MODES = ["stated", "inferred", "unknown"] as const; export type ProvenanceMode = (typeof PROVENANCE_MODES)[number]; +/** + * Temporal ranking class stored on knowledge.version.temporal_class. + * See docs/TEMPORAL.md. + */ +export const TEMPORAL_CLASSES = [ + "event", + "deadline", + "state", + "lesson", +] as const; +export type TemporalClass = (typeof TEMPORAL_CLASSES)[number]; + /** Build an arktype union string from a const string array. */ export function arktypeStringUnion( values: readonly string[], diff --git a/src/core/hybrid-search.test.ts b/src/core/hybrid-search.test.ts index 7637dac..c2beb12 100644 --- a/src/core/hybrid-search.test.ts +++ b/src/core/hybrid-search.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test"; import { BOOST_MULTIPLIER_MAX, BOOST_MULTIPLIER_MIN, + DEADLINE_LOOKAHEAD_MS, MAX_BATCH_QUERIES, RECENCY_HALF_LIFE_MS, authorityBoostMultiplier, @@ -11,6 +12,7 @@ import { isBatchQueriesWithinBound, normalizeScoresToUnit, recencyBoostMultiplier, + temporalRecencyMultiplier, toRankedCandidates, } from "./hybrid-search.ts"; @@ -194,3 +196,80 @@ describe("recencyBoostMultiplier", () => { ); }); }); + +describe("temporalRecencyMultiplier", () => { + const now = new Date("2026-07-20T00:00:00.000Z"); + + test("event matches recencyBoostMultiplier", () => { + const oneHalfLifeAgo = new Date(now.getTime() - RECENCY_HALF_LIFE_MS); + expect( + temporalRecencyMultiplier({ + temporalClass: "event", + occurredAt: oneHalfLifeAgo, + validUntil: null, + now, + }), + ).toBeCloseTo(recencyBoostMultiplier(oneHalfLifeAgo, now), 10); + }); + + test("state and lesson are neutral regardless of age", () => { + const old = new Date("2020-01-01T00:00:00.000Z"); + for (const temporalClass of ["state", "lesson"] as const) { + expect( + temporalRecencyMultiplier({ + temporalClass, + occurredAt: old, + validUntil: null, + now, + }), + ).toBe(1.0); + } + }); + + test("deadline far out is neutral", () => { + const far = new Date(now.getTime() + DEADLINE_LOOKAHEAD_MS * 2); + expect( + temporalRecencyMultiplier({ + temporalClass: "deadline", + occurredAt: now, + validUntil: far, + now, + }), + ).toBe(1.0); + }); + + test("deadline at expiry approaches the urgency ceiling (~1.3)", () => { + const almostDue = new Date(now.getTime() + 1); + const mult = temporalRecencyMultiplier({ + temporalClass: "deadline", + occurredAt: now, + validUntil: almostDue, + now, + }); + expect(mult).toBeGreaterThan(1.25); + expect(mult).toBeLessThanOrEqual(BOOST_MULTIPLIER_MAX); + }); + + test("expired deadline falls to the boost floor (still retrievable)", () => { + const past = new Date(now.getTime() - 1000); + expect( + temporalRecencyMultiplier({ + temporalClass: "deadline", + occurredAt: now, + validUntil: past, + now, + }), + ).toBe(BOOST_MULTIPLIER_MIN); + }); + + test("deadline with null validUntil is neutral", () => { + expect( + temporalRecencyMultiplier({ + temporalClass: "deadline", + occurredAt: now, + validUntil: null, + now, + }), + ).toBe(1.0); + }); +}); diff --git a/src/core/hybrid-search.ts b/src/core/hybrid-search.ts index 76a0579..4aeda1d 100644 --- a/src/core/hybrid-search.ts +++ b/src/core/hybrid-search.ts @@ -155,3 +155,41 @@ export function recencyBoostMultiplier( const decay = Math.pow(2, -ageMs / halfLifeMs); return clampBoostMultiplier(BOOST_BASE + BOOST_SPAN * decay); } + +// How far ahead of valid_until a deadline starts ramping urgency (neutral +// before this window; approaches the boost ceiling at the deadline). +export const DEADLINE_LOOKAHEAD_MS = 7 * 24 * 60 * 60 * 1000; + +export type TemporalRecencyInput = { + temporalClass: "event" | "deadline" | "state" | "lesson"; + occurredAt: Date; + validUntil: Date | null; + now: Date; + halfLifeMs?: number; +}; + +/** + * Recency prior by temporal class (docs/TEMPORAL.md): + * - event: exponential decay from occurred_at (existing half-life) + * - deadline: neutral far out; urgency ramp in lookahead before valid_until; + * floor after expiry (still history-retrievable, not deleted) + * - state / lesson: no decay while active (superseded rows are status-filtered) + */ +export function temporalRecencyMultiplier(input: TemporalRecencyInput): number { + const halfLifeMs = input.halfLifeMs ?? RECENCY_HALF_LIFE_MS; + switch (input.temporalClass) { + case "event": + return recencyBoostMultiplier(input.occurredAt, input.now, halfLifeMs); + case "state": + case "lesson": + return 1.0; + case "deadline": { + if (input.validUntil === null) return 1.0; + const remaining = input.validUntil.getTime() - input.now.getTime(); + if (remaining <= 0) return BOOST_MULTIPLIER_MIN; + if (remaining >= DEADLINE_LOOKAHEAD_MS) return 1.0; + const urgency = 1 - remaining / DEADLINE_LOOKAHEAD_MS; + return clampBoostMultiplier(1.0 + BOOST_SPAN * urgency * 0.5); + } + } +} diff --git a/src/core/schemas/adapted-document.ts b/src/core/schemas/adapted-document.ts index 5f7fce7..10642b4 100644 --- a/src/core/schemas/adapted-document.ts +++ b/src/core/schemas/adapted-document.ts @@ -3,6 +3,7 @@ import { CreatedByKindSchema, LineageClassSchema, ProvenanceModeSchema, + TemporalClassSchema, } from "./document.ts"; import { MemoryEdgeHintSchema } from "./entity-edge.ts"; import { AuthoritySourceClassSchema } from "../authority.ts"; @@ -81,6 +82,9 @@ export const AdaptedDocumentSchema = type({ "hasSocialSignal?": "boolean", "lineageClass?": LineageClassSchema, "provenance?": ProvenanceModeSchema, + "temporalClass?": TemporalClassSchema, + "validFrom?": "string", + "validUntil?": "string", contentHash: "string", }); export type AdaptedDocument = typeof AdaptedDocumentSchema.infer; diff --git a/src/core/schemas/document.test.ts b/src/core/schemas/document.test.ts index fdb3bc8..b270b62 100644 --- a/src/core/schemas/document.test.ts +++ b/src/core/schemas/document.test.ts @@ -59,6 +59,7 @@ describe("MemoryVersionSchema", () => { created_by_kind: "human", provenance: "stated", source_class: "native", + temporal_class: "event", }; const out = MemoryVersionSchema(fixture); expect(out instanceof type.errors ? out.summary : out).toEqual(fixture); @@ -82,6 +83,7 @@ describe("MemoryVersionSchema", () => { created_by_kind: "human", provenance: "stated", source_class: "native", + temporal_class: "event", }); expect(out instanceof type.errors).toBe(true); }); diff --git a/src/core/schemas/document.ts b/src/core/schemas/document.ts index 50827e7..9cd9657 100644 --- a/src/core/schemas/document.ts +++ b/src/core/schemas/document.ts @@ -2,6 +2,7 @@ import { type } from "arktype"; import { LINEAGE_CLASSES, PROVENANCE_MODES, + TEMPORAL_CLASSES, arktypeStringUnion, } from "../enums.ts"; @@ -23,6 +24,12 @@ export const ProvenanceModeSchema = type( ); export type ProvenanceMode = typeof ProvenanceModeSchema.infer; +export const TemporalClassSchema = type( + arktypeStringUnion(TEMPORAL_CLASSES) as + "'event'|'deadline'|'state'|'lesson'", +); +export type TemporalClass = typeof TemporalClassSchema.infer; + // The stable logical row for a captured source, deduped on (tenant_id, // adapter, external_ref). Document access is grant tags only. export const MemoryDocumentSchema = type({ @@ -41,6 +48,9 @@ export type MemoryDocument = typeof MemoryDocumentSchema.infer; // The versioned body of a document. Chunks belong to a version_id, never // reused across versions. +// occurred_at is effective time the content refers to (event time / state +// effective time / deadline establishment). ingested_at is when the plane +// learned it. Validity window is optional (deadline/state claims). export const MemoryVersionSchema = type({ id: "string", tenant_id: "string", @@ -59,5 +69,8 @@ export const MemoryVersionSchema = type({ "generator_agent_id?": "string", provenance: ProvenanceModeSchema, source_class: LineageClassSchema, + temporal_class: TemporalClassSchema, + "valid_from?": "string | null", + "valid_until?": "string | null", }); export type MemoryVersion = typeof MemoryVersionSchema.infer; diff --git a/src/db/schema.ts b/src/db/schema.ts index 40d572b..70b6566 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -75,6 +75,10 @@ export const memoryVersion = memorySchema.table( // How content was obtained: stated (asserted), inferred (derived claim), // unknown (legacy / unset). Orthogonal to created_by_kind and source_class. provenance: text("provenance").notNull().default("unknown"), + // Ranking temporal class + optional validity window (docs/TEMPORAL.md). + temporalClass: text("temporal_class").notNull().default("event"), + validFrom: timestamp("valid_from"), + validUntil: timestamp("valid_until"), rawCaptureId: text("raw_capture_id").references(() => rawCapture.id), // Replay-generation tag — 'live' for the normal /capture path; a replay tags every // version it writes with its own transform_run id instead, so a replayed diff --git a/src/services/capture.ts b/src/services/capture.ts index 3a451d9..715215b 100644 --- a/src/services/capture.ts +++ b/src/services/capture.ts @@ -92,6 +92,18 @@ function deriveProvenance(plan: CapturePlan): string { return plan.document.provenance ?? "stated"; } +function deriveTemporalClass(plan: CapturePlan): string { + if (plan.document.temporalClass) return plan.document.temporalClass; + // Distilled claims default to state ranking; raw captures to event. + if ((plan.document.provenance ?? "stated") === "inferred") return "state"; + return "event"; +} + +function parseOptionalDate(value: string | undefined): Date | null { + if (!value) return null; + return new Date(value); +} + async function insertVersion( tx: Tx, input: CaptureInput, @@ -125,6 +137,9 @@ async function insertVersion( hasSocialSignal: authoritySignals.hasSocialSignal, sourceClass: deriveLineageClass(plan), provenance: deriveProvenance(plan), + temporalClass: deriveTemporalClass(plan), + validFrom: parseOptionalDate(plan.document.validFrom), + validUntil: parseOptionalDate(plan.document.validUntil), rawCaptureId: opts.rawCaptureId, generation: opts.generation, }); diff --git a/src/services/search.test.ts b/src/services/search.test.ts index e43cae9..942263f 100644 --- a/src/services/search.test.ts +++ b/src/services/search.test.ts @@ -26,6 +26,8 @@ function candidate(overrides: Partial = {}): CandidateRow { rank: 1, occurredAt: new Date("2026-01-01T00:00:00Z"), authority: 0.5, + temporalClass: "event", + validUntil: null, ...overrides, }; } diff --git a/src/services/search.ts b/src/services/search.ts index 636df0a..81eb300 100644 --- a/src/services/search.ts +++ b/src/services/search.ts @@ -34,7 +34,7 @@ import { DEFAULT_OVERFETCH_MULTIPLIER, fuseRrf, normalizeScoresToUnit, - recencyBoostMultiplier, + temporalRecencyMultiplier, RECENCY_HALF_LIFE_MS, toRankedCandidates, type DegradeFlag, @@ -128,6 +128,10 @@ export interface CandidateRow { // The version's stored 0..1 authority score (computed at capture time; // never recomputed here). authority: number; + // Temporal ranking class + validity (docs/TEMPORAL.md). Defaults applied + // when a row predates the temporal migration (should not happen after 0004). + temporalClass: "event" | "deadline" | "state" | "lesson"; + validUntil: Date | null; } export function snippet(text: string, maxLen = 240): string { @@ -399,6 +403,8 @@ export async function fetchLexicalCandidates( rank: rankExpr, occurredAt: memoryVersion.occurredAt, authority: memoryVersion.authority, + temporalClass: memoryVersion.temporalClass, + validUntil: memoryVersion.validUntil, }) .from(memoryChunk) .innerJoin( @@ -527,7 +533,8 @@ export async function fetchDenseCandidates( kv.version AS version, kv.status AS status, kd.title AS title, kd.kind AS kind, kd.adapter AS adapter, kd.external_ref AS external_ref, kv.created_by_kind AS created_by_kind, kv.generator_agent_id AS generator_agent_id, - c.text AS snippet_text, kv.occurred_at AS occurred_at, kv.authority AS authority + c.text AS snippet_text, kv.occurred_at AS occurred_at, kv.authority AS authority, + kv.temporal_class AS temporal_class, kv.valid_until AS valid_until FROM ${activeTable.tableName} e JOIN "memory"."chunk" c ON c.id = e.chunk_id JOIN "memory"."version" kv ON kv.id = c.version_id @@ -589,6 +596,10 @@ export async function fetchDenseCandidates( rank: 0, occurredAt: new Date(row["occurred_at"] as string), authority: row["authority"] as number, + temporalClass: (row["temporal_class"] as CandidateRow["temporalClass"]) ?? "event", + validUntil: row["valid_until"] + ? new Date(row["valid_until"] as string) + : null, })); } @@ -651,7 +662,13 @@ function applyBoosts( return rows.map((row, index) => { const normScore = normalized[index] ?? 0; const authorityMult = authorityBoostMultiplier(row.authority); - const recencyMult = recencyBoostMultiplier(row.occurredAt, now, recencyHalfLifeMs); + const recencyMult = temporalRecencyMultiplier({ + temporalClass: row.temporalClass, + occurredAt: row.occurredAt, + validUntil: row.validUntil, + now, + halfLifeMs: recencyHalfLifeMs, + }); return { row, finalScore: normScore * authorityMult * recencyMult }; }); } diff --git a/src/services/timeline.test.ts b/src/services/timeline.test.ts index b623560..5967c6d 100644 --- a/src/services/timeline.test.ts +++ b/src/services/timeline.test.ts @@ -3,6 +3,7 @@ import { createInMemoryGrantStore } from "@intx/authz"; import { PgDialect } from "drizzle-orm/pg-core"; import { + activeTimelineVersionJoin, filterTimelineRows, timelineWhere, type TimelineRow, @@ -96,3 +97,21 @@ describe("timelineWhere", () => { expect(sql).not.toContain("visibility_principal_ids"); }); }); + +describe("activeTimelineVersionJoin", () => { + it("filters active status and live generation by default", () => { + const { sql, params } = dialect.sqlToQuery(activeTimelineVersionJoin()!); + expect(sql).toContain("status"); + expect(sql).toContain("generation"); + expect(params).toContain("active"); + expect(params).toContain("live"); + }); + + it("accepts a replay generation tag", () => { + const { params } = dialect.sqlToQuery( + activeTimelineVersionJoin("replay_run_1")!, + ); + expect(params).toContain("replay_run_1"); + expect(params).not.toContain("live"); + }); +}); diff --git a/src/services/timeline.ts b/src/services/timeline.ts index 4788203..4f68f70 100644 --- a/src/services/timeline.ts +++ b/src/services/timeline.ts @@ -10,6 +10,7 @@ import { and, desc, eq, sql } from "drizzle-orm"; import type { ConditionRegistry, GrantStore } from "@intx/authz"; import { canAccessDocument } from "../grant-tags.ts"; +import { LIVE_GENERATION } from "../core/generation.ts"; import type { Db } from "../db/client.ts"; import { memoryDocument, memoryVersion } from "../db/schema.ts"; @@ -31,6 +32,11 @@ export type ListTimelineParams = { /** Host grant store — required for non-creator document access. */ grants?: GrantStore; conditionRegistry?: ConditionRegistry; + /** + * Replay-generation tag. Defaults to live so staged replay versions never + * appear in the default timeline (matches hybrid search). + */ + generation?: string; }; export type TimelineRow = { @@ -55,6 +61,20 @@ export function timelineWhere(tenantId: string) { return eq(memoryDocument.tenantId, tenantId); } +/** + * Active-version join for timeline: status + generation (live by default). + * Exported so tests can assert the generation predicate without a live DB. + */ +export function activeTimelineVersionJoin( + generation: string = LIVE_GENERATION, +) { + return and( + eq(memoryVersion.documentId, memoryDocument.id), + eq(memoryVersion.status, "active"), + eq(memoryVersion.generation, generation), + ); +} + /** * Filter raw timeline rows to those the principal may see under grant tags. */ @@ -108,6 +128,7 @@ export async function filterTimelineRows( /** * List recent document events for a tenant, filtered by grant-tag access. + * Only active versions in the requested generation (default live) appear. */ export async function listTimelineEvents( params: ListTimelineParams, @@ -117,6 +138,7 @@ export async function listTimelineEvents( MAX_LIMIT, ); const fetchLimit = Math.min(limit * TIMELINE_OVERFETCH, MAX_LIMIT * TIMELINE_OVERFETCH); + const generation = params.generation ?? LIVE_GENERATION; const rows = await params.db .select({ @@ -129,13 +151,7 @@ export async function listTimelineEvents( accessTags: memoryDocument.accessTags, }) .from(memoryDocument) - .innerJoin( - memoryVersion, - and( - eq(memoryVersion.documentId, memoryDocument.id), - eq(memoryVersion.status, "active"), - ), - ) + .innerJoin(memoryVersion, activeTimelineVersionJoin(generation)) .where(timelineWhere(params.tenantId)) .orderBy(desc(memoryVersion.occurredAt)) .limit(fetchLimit); From af24dc078549d89d03764e14c618c49e09d8434f Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 10 Aug 2026 05:17:21 -0700 Subject: [PATCH 03/19] feat: staged transform/replay and share grant materialization CL-5872: expose transform plane methods with staged promote/demote; split ensureEmbedModel vs activateEmbedModel so replay never steals live dense search; generation-scoped dense table resolution. CL-5873: materialize peer share grants via WritableGrantStore on memory.doc tags; merge MEMORY_SHARE_CONDITION_REGISTRY; write-narrow- then-widen helpers and AUTHZ docs. --- IMPLEMENTATION.md | 94 +++++--- docs/AUTHZ-DOCUMENT-ACCESS.md | 34 ++- migrations/0005_transform_promote.sql | 7 + src/core/embed-model-registry.test.ts | 65 ++++++ src/core/embed-model-registry.ts | 73 +++++- src/db/schema.ts | 3 + src/grant-tags.ts | 3 +- src/index.ts | 44 ++++ src/memory.ts | 307 ++++++++++++++++++++------ src/ports/fakes.ts | 8 + src/ports/types.ts | 9 + src/ports/writable-grant-store.ts | 51 +++++ src/services/capture.ts | 35 +-- src/services/search.ts | 63 ++++-- src/services/share-grants.test.ts | 178 +++++++++++++++ src/services/share-grants.ts | 161 ++++++++++++++ src/services/transform.ts | 192 +++++++++++++++- 17 files changed, 1176 insertions(+), 151 deletions(-) create mode 100644 migrations/0005_transform_promote.sql create mode 100644 src/ports/writable-grant-store.ts create mode 100644 src/services/share-grants.test.ts create mode 100644 src/services/share-grants.ts diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md index 53496e2..ffcd2f1 100644 --- a/IMPLEMENTATION.md +++ b/IMPLEMENTATION.md @@ -232,14 +232,28 @@ See `docs/TEMPORAL.md`. On `memory.version`: Search multiplies fused scores by `temporalRecencyMultiplier` (class-aware). Timeline and search both filter `generation` (default live) so replay rows never leak into live views. ### `memory_embed_model` -Per-tenant registry of which embed model is currently active, and the -dimensionality it was discovered at (`discoverModelDims`/`probeEmbedDims` — -dims are **never** hard-coded, always probed live against the endpoint). -Unique on `(tenant_id, model_key)`, where `model_key` is -`sha256(baseUrl|modelId).slice(0,16)` (`computeModelKey`). "Active" means the -most-recently-`updated_at` row with `status = 'active'` for that tenant -(`resolveActiveEmbedTable`) — there is no per-generation embed-model scoping -(see "Known limitation" under Raw + replay below). +Per-tenant registry of embed models and their discovered dimensionality +(`discoverModelDims`/`probeEmbedDims` — dims are **never** hard-coded, always +probed live against the endpoint). Unique on `(tenant_id, model_key)`, where +`model_key` is `sha256(baseUrl|modelId).slice(0,16)` (`computeModelKey`). + +Two write paths (CL-5872): + +- **`ensureEmbedModel`** — upserts the registry row with `status = 'ready'`, + creates the per-model table + indexes. **Never** flips the tenant's active + dense table. Used by staged `runTransform` when the config's embed model + differs from live. +- **`activateEmbedModel`** — `ensure` then `UPDATE … SET status = 'active'`. + Used only by the **live capture** path and by **`promoteGeneration`** when + cutover should make the generation's embed model serve live dense search. + +"Active" for live dense search is the most-recently-`updated_at` row with +`status = 'active'` (`resolveActiveEmbedTable`). Replay dense search uses +`resolveEmbedTableByModelKey` against the owning transform config's embed +`model_key` — ready or active — so staged generations never steal live. + +Optional `archived_live_generation` on `transform_run` records which generation +held live rows before promote (for demote/rollback). ### Dynamic per-model vector tables: `memory_embedding_` Not in `db/schema.ts` (no fixed shape — dimensionality varies by model) and @@ -403,9 +417,11 @@ search); otherwise it throws `MemorySearchInputError` (400). `kinds` and/or `entityIds` (via a sub-select against `memory_edge`). Overfetches up to `overfetchLimit` rows, non-deduped, per-chunk. 3. **Dense channel** — `fetchDenseCandidates`: embeds the query - (`embedTexts`), resolves the tenant's single active embedding table - (`resolveActiveEmbedTable` — **not** generation-scoped, see limitation - below), runs a raw-SQL cosine-distance ANN query via `cosineDistanceExpr` + (`embedTexts`), resolves the dense table: + - live generation → `resolveActiveEmbedTable` (tenant active model) + - staged generation → `resolveEmbedTableByModelKey` for the transform + config's embed model (ready or active; never activates) + runs a raw-SQL cosine-distance ANN query via `cosineDistanceExpr` (`e.embedding <=> $vector` up to 2000 dims, or the matching `(e.embedding::halfvec(N)) <=> $vector::halfvec(N)` expression above that so the halfvec HNSW index is used) @@ -481,28 +497,30 @@ writes to. A `transform_config` is a named/versioned recipe own trust boundary, never a blind `JSON.parse`), then calls `deriveFromRawCapture` with the config's own chunker (`chunkTokenRecursive` with the config's caps) and embed client config, - targeting the run's `generation` — never the live one. + targeting the run's `generation` — never the live one. Embed tables are + **ensured** (`ensureEmbedModel`), never activated, so live dense search is + untouched until an explicit promote. 5. On completion, updates the run row: `status: 'completed'`, `rawCount`, `versionCount`. On any exception mid-loop, catches it, logs it, and marks the run `'failed'` with `error` set — `runTransform` itself never throws to its caller; callers always get a run summary. 6. `resolveGenerationSearchParams` is how `hybridSearch` later maps a generation back to its config's search-tuning knobs (authority weight, - recency half-life, MMR λ, overfetch, rerank config). - -**Documented limitation — per-generation embed-model isolation does not -exist.** `resolveActiveEmbedTable` picks the tenant's single -most-recently-`updated_at` active model, with no `generation` argument at -all. If a replay's `transform_config.embed` points at a *different* -`(baseUrl, modelId)` than the live capture path currently uses, running that -replay makes its model the tenant's active dense-channel table for **every** -generation's search, including live's, from that point forward. The `search.ts` -comment on `fetchDenseCandidatesArgs.generation` states this explicitly; -`transform.ts`'s replay test in `e2e.integration.test.ts` sidesteps it by -reusing the exact same embed endpoint/model as the live capture. Scoping -activation per-generation is out of scope for the current replay-pipeline -implementation — callers replaying under a different embed model should do -so knowing it will flip the tenant's live dense channel too. + recency half-life, MMR λ, overfetch, rerank config) **and** dense + `modelKey` for generation-scoped table resolution. + +**Promote / demote (staged cutover):** + +- `promoteGeneration` — swaps `status` so the staged generation becomes + `active` under `generation = 'live'` semantics for search (prior live is + archived; `archived_live_generation` records the previous id for rollback). + Optionally activates the generation's embed model so live dense matches + the promoted embeddings. +- `demoteGeneration` — restores the archived live generation. + +Plane methods (engine DocumentStore only): `createTransformConfig`, +`listTransformConfigs`, `runTransform`, `promoteGeneration`, +`demoteGeneration` on `Memory`. Custom/fake stores omit these methods. ## Mounted routes @@ -529,8 +547,26 @@ mounted routes with install env (`memoryBaseUrl`, `memoryTenantId`, needs `memory:add` and/or `memory:search` grants; Bearer token only (no session cookie path); tool results are JSON strings; pass `AbortSignal` if you need hang protection — the client has no default timeout. OpenAPI→MCP remains an optional -host bridge. The plane surface is only `add` / `search` / `list` (plus `close`); -inference stays on the host. +host bridge. The plane surface is `add` / `search` / `list` / `close`, plus +optional transform methods when backed by the engine DocumentStore +(`createTransformConfig`, `listTransformConfigs`, `runTransform`, +`promoteGeneration`, `demoteGeneration`). Inference stays on the host. + +### Share materialization (CL-5873) + +`share.principals` on `add` still mints owner tags, and when the host grant +store implements `WritableGrantStore.putGrant`: + +1. Appends `memory.doc:` to the document's `access_tags`. +2. Writes one allow/`search` grant per peer on that resource, origin + `system`, with `conditions.memoryShare` audit payload. +3. `resolveGrantConfig` merges `MEMORY_SHARE_CONDITION_REGISTRY` so those + condition keys are not fail-closed-skipped by `@intx/authz`. + +Without a writable store: tags only + warn log (peers need host grants). +Audience widening uses `splitAudienceWiden` (write-narrow-then-widen) and +`shareWidenReceipt` on version attributes after source-owner approval. +Ask-on-read remains design-only (fail-closed). diff --git a/docs/AUTHZ-DOCUMENT-ACCESS.md b/docs/AUTHZ-DOCUMENT-ACCESS.md index 6f1ee3e..fad88c9 100644 --- a/docs/AUTHZ-DOCUMENT-ACCESS.md +++ b/docs/AUTHZ-DOCUMENT-ACCESS.md @@ -78,18 +78,28 @@ There is **no** `share.private` key. Owner-only is the default when `share` is o Tag minting is **not** grant minting. For peer share to work in product: 1. When Alice adds with `share: { principals: ["bob"] }`, the document is tagged - `memory.owner:alice` and `memory.owner:bob`. -2. Bob sees it only if the host has granted Bob `search` on `memory.owner:bob` - (or a pattern that matches). **Recommended host bootstrap:** every principal - receives `search` (and optionally `add` side-effects as you prefer) on - `memory.owner:` at signup, or a single pattern grant such as - `memory.owner:*` only if that matches your tenancy model. -3. Space/tenant tags work the same way: host must issue grants on - `memory.space:eng` / `memory.tenant:` for non-creators to match. - -Without (2), `share.principals` is a silent no-op for peers (fail-closed; looks -like empty search). Document this in host mount guides — do not reintroduce a -document mini-ACL in this package. + `memory.owner:alice` and (after insert) `memory.doc:`. +2. When the host grant store implements `WritableGrantStore.putGrant`, memory + **materializes** an allow/`search` grant for Bob on `memory.doc:` + (origin `system`, conditions carry `memoryShare` / `sharedBy` / `sourceVersionId` + for audit). Peers can then pass `canAccessDocument` without a separate + bootstrap grant on `memory.owner:bob`. +3. Without a writable grant store, tags alone are written and peers still need + host-side grants (fail-closed empty search). Log warns on this path. +4. `share.tenant` / `share.tags` still only mint tags — hosts issue role/pattern + grants on those resources (no auto-principal grants). + +### Audience widening (write-narrow-then-widen) + +Distiller / claim writes that propose tags **beyond** the source document's +audience must not silently widen: + +1. `splitAudienceWiden(sourceTags, proposed)` → write with `allowed` only. +2. After **source-owner** approval, append `needsApproval` tags and materialize + any peer grants; store `shareWidenReceipt` on version attributes. + +Ask-on-read (`authorize` effect `"ask"`) remains **fail-closed** in +`canAccessDocument` (design-only; no flag in v1). **Removed:** `visibility: { mode: private|principals|tenant }`, `blockPrincipalIds`, product `acl.mode` / `acl.allow` / `acl.block` as security. diff --git a/migrations/0005_transform_promote.sql b/migrations/0005_transform_promote.sql new file mode 100644 index 0000000..40ffcaa --- /dev/null +++ b/migrations/0005_transform_promote.sql @@ -0,0 +1,7 @@ +-- Promote / demote bookkeeping for staged transform generations (CL-5872). +-- archived_live_generation holds the generation tag assigned to the prior +-- live corpus during promote, so demote can swap back without data loss. + +ALTER TABLE "memory"."transform_run" + ADD COLUMN IF NOT EXISTS "archived_live_generation" text, + ADD COLUMN IF NOT EXISTS "promoted_at" timestamp with time zone; diff --git a/src/core/embed-model-registry.test.ts b/src/core/embed-model-registry.test.ts index 19cd032..30ab10e 100644 --- a/src/core/embed-model-registry.test.ts +++ b/src/core/embed-model-registry.test.ts @@ -8,9 +8,11 @@ import { discoverModelDims, EMBED_TABLE_NAME_PATTERN, embeddingTableName, + ensureEmbedModel, type EmbedRegistrySqlClient, HALFVEC_INDEX_MAX_DIMS, resolveActiveEmbedTable, + resolveEmbedTableByModelKey, VECTOR_INDEX_MAX_DIMS, } from "./embed-model-registry.ts"; @@ -272,6 +274,69 @@ describe("resolveActiveEmbedTable", () => { tableName: `"memory"."embedding_${modelKey}"`, dims: 768, modelId: baseConfig.modelId, + modelKey, }); }); }); + +describe("ensureEmbedModel", () => { + it("inserts with status='ready' (never active) so live dense search is untouched", async () => { + const { client, queries } = createMockClient(); + await ensureEmbedModel(client, "tenant-1", baseConfig, fixtureFetch(768)); + + const insertQuery = queries.find((q) => + q.sql.includes('INSERT INTO "memory"."embed_model"'), + ); + expect(insertQuery?.sql).toContain("'ready'"); + expect(insertQuery?.sql).not.toContain("'active'"); + + // No UPDATE ... SET status='active' issued by ensure. + expect( + queries.some((q) => q.sql.includes("SET status = 'active'")), + ).toBe(false); + }); + + it("still creates the per-model table + indexes (table usable on replay)", async () => { + const { client, queries } = createMockClient(); + const result = await ensureEmbedModel(client, "tenant-1", baseConfig, fixtureFetch(768)); + const createTableQuery = queries.find((q) => q.sql.includes("CREATE TABLE IF NOT EXISTS")); + expect(createTableQuery?.sql).toContain(result.tableName); + expect(queries.some((q) => q.sql.includes("USING hnsw"))).toBe(true); + }); +}); + +describe("activateEmbedModel (split)", () => { + it("ensures then issues UPDATE ... SET status='active', updated_at=now()", async () => { + const { client, queries } = createMockClient(); + await activateEmbedModel(client, "tenant-1", baseConfig, fixtureFetch(768)); + + const updateQuery = queries.find((q) => + q.sql.includes("SET status = 'active'"), + ); + expect(updateQuery).toBeDefined(); + expect(updateQuery?.sql).toContain("updated_at = now()"); + }); +}); + +describe("resolveEmbedTableByModelKey", () => { + it("returns null when no row for this model_key", async () => { + const client: EmbedRegistrySqlClient = { query: () => Promise.resolve([]) }; + expect(await resolveEmbedTableByModelKey(client, "tenant-1", "abcdef0123456789")).toBeNull(); + }); + + it("returns the table info regardless of status (ready or active)", async () => { + const modelKey = "abcdef0123456789"; + const client: EmbedRegistrySqlClient = { + query: () => + Promise.resolve([{ model_key: modelKey, model_id: "m", dims: 512 }]), + }; + const result = await resolveEmbedTableByModelKey(client, "tenant-1", modelKey); + expect(result?.dims).toBe(512); + expect(result?.tableName).toBe(`"memory"."embedding_${modelKey}"`); + }); + + it("rejects an invalid model_key format", async () => { + const client: EmbedRegistrySqlClient = { query: () => Promise.resolve([]) }; + expect(() => resolveEmbedTableByModelKey(client, "tenant-1", "not-a-key")).toThrow(); + }); +}); diff --git a/src/core/embed-model-registry.ts b/src/core/embed-model-registry.ts index 147e125..e3079ca 100644 --- a/src/core/embed-model-registry.ts +++ b/src/core/embed-model-registry.ts @@ -111,7 +111,15 @@ export interface ActivateEmbedModelResult { modelKey: string; } -export async function activateEmbedModel( +/** + * Ensure the per-model embedding table and registry row exist without making + * this model the tenant's active dense-search target. + * + * Used by transform/replay so a staged embed model flip never steals + * `resolveActiveEmbedTable` from live. Idempotent: CREATE TABLE IF NOT EXISTS + * + registry upsert that never promotes status to `active`. + */ +export async function ensureEmbedModel( client: EmbedRegistrySqlClient, tenantId: string, config: EmbedClientConfig, @@ -122,11 +130,14 @@ export async function activateEmbedModel( const tableName = embeddingTableName(modelKey); const bare = embeddingTableBareName(modelKey); + // status stays 'ready' on insert; ON CONFLICT never overwrites an existing + // status (so an already-active model remains active) and never bumps + // updated_at (so a ready model cannot leapfrog resolveActiveEmbedTable). await client.query( `INSERT INTO "memory"."embed_model" (id, tenant_id, model_key, model_id, dims, status, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, 'active', now(), now()) + VALUES ($1, $2, $3, $4, $5, 'ready', now(), now()) ON CONFLICT (tenant_id, model_key) - DO UPDATE SET model_id = EXCLUDED.model_id, dims = EXCLUDED.dims, updated_at = now()`, + DO UPDATE SET model_id = EXCLUDED.model_id, dims = EXCLUDED.dims`, [randomUUID(), tenantId, modelKey, config.modelId, dims], ); @@ -148,7 +159,7 @@ export async function activateEmbedModel( ); // Composite over bare tenant_id: the read path filters tenant_id plus a // chunk_id set; this is a B-tree membership filter (fetchChunkVectors also - // selects embedding). Runs on every activation so a pre-FK table still gets + // selects embedding). Runs on every ensure so a pre-FK table still gets // the index. await client.query( `CREATE INDEX IF NOT EXISTS ${bare}_tenant_chunk_idx ON ${tableName} (tenant_id, chunk_id)`, @@ -178,7 +189,7 @@ export async function activateEmbedModel( // cosineDistanceExpr emits for these dims, or the planner ignores the // index. No ivfflat fallback on this path: halfvec requires // pgvector >= 0.7.0 and every such release has hnsw, so a failure here - // means the extension is too old for halfvec at all — let activation + // means the extension is too old for halfvec at all — let ensure // fail loudly at this boundary rather than accept an unindexable model. // IF NOT EXISTS also retrofits the index onto a table created before // halfvec support existed; on a large populated table this build can @@ -192,10 +203,32 @@ export async function activateEmbedModel( return { tableName, dims, modelId: config.modelId, modelKey }; } +/** + * Ensure the table exists, then promote this model to the tenant's active + * dense-search target (`status='active'`, `updated_at=now()`). Live capture + * uses this path; transform/replay must use `ensureEmbedModel` only. + */ +export async function activateEmbedModel( + client: EmbedRegistrySqlClient, + tenantId: string, + config: EmbedClientConfig, + fetchImpl: typeof fetch = fetch, +): Promise { + const result = await ensureEmbedModel(client, tenantId, config, fetchImpl); + await client.query( + `UPDATE "memory"."embed_model" + SET status = 'active', updated_at = now() + WHERE tenant_id = $1 AND model_key = $2`, + [tenantId, result.modelKey], + ); + return result; +} + export interface ActiveEmbedTable { tableName: string; dims: number; modelId: string; + modelKey?: string; } export async function resolveActiveEmbedTable( @@ -217,5 +250,35 @@ export async function resolveActiveEmbedTable( tableName: embeddingTableName(modelKey), dims: row.dims as number, modelId: row.model_id as string, + modelKey, + }; +} + +/** + * Resolve any registered embed table by model_key (active or ready). + * Used when searching a non-live generation whose transform_config embeds + * with a model that is not the tenant's active one. + */ +export async function resolveEmbedTableByModelKey( + client: EmbedRegistrySqlClient, + tenantId: string, + modelKey: string, +): Promise { + if (!/^[a-f0-9]{16}$/.test(modelKey)) { + throw new Error(`resolveEmbedTableByModelKey: invalid modelKey "${modelKey}"`); + } + const rows = await client.query( + `SELECT model_key, model_id, dims FROM "memory"."embed_model" + WHERE tenant_id = $1 AND model_key = $2 + LIMIT 1`, + [tenantId, modelKey], + ); + const row = rows[0]; + if (!row) return null; + return { + tableName: embeddingTableName(modelKey), + dims: row.dims as number, + modelId: row.model_id as string, + modelKey, }; } diff --git a/src/db/schema.ts b/src/db/schema.ts index 70b6566..8c20b34 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -234,6 +234,9 @@ export const transformRun = memorySchema.table( error: text("error"), createdAt: timestamp("created_at").notNull().defaultNow(), completedAt: timestamp("completed_at"), + /** Generation tag assigned to the prior live corpus on promote (for demote). */ + archivedLiveGeneration: text("archived_live_generation"), + promotedAt: timestamp("promoted_at"), }, (t) => [ uniqueIndex("transform_run_generation_uniq").on(t.generation), diff --git a/src/grant-tags.ts b/src/grant-tags.ts index 3affe4b..36677fb 100644 --- a/src/grant-tags.ts +++ b/src/grant-tags.ts @@ -6,7 +6,8 @@ * - Capability checks (add/search on `memory`) live on the HTTP mount. * - Document access: creator always sees own docs; otherwise any `accessTag` * that `authorize(…, tag, "search")` allows. - * - Share sugars only mint tags — they never write grants. + * - Share sugars mint tags; peer grants are materialized separately when the + * host provides a WritableGrantStore (see services/share-grants.ts). */ import { authorize } from "@intx/authz"; import type { ConditionRegistry, GrantStore } from "@intx/authz"; diff --git a/src/index.ts b/src/index.ts index 93eba33..a5dec49 100644 --- a/src/index.ts +++ b/src/index.ts @@ -74,6 +74,50 @@ export { createFakeSourceProvider, } from "./ports/fakes.ts"; +export type { WritableGrantStore } from "./ports/writable-grant-store.ts"; +export { + createInMemoryWritableGrantStore, + isWritableGrantStore, +} from "./ports/writable-grant-store.ts"; + +// Share materialization (CL-5873) +export { + buildShareGrants, + documentTag, + materializeShareGrants, + MEMORY_SHARE_CONDITION_KEY, + MEMORY_SHARE_CONDITION_REGISTRY, + shareWidenReceipt, + splitAudienceWiden, + type MaterializeShareGrantsInput, + type MemoryShareCondition, + type ShareWidenReceipt, +} from "./services/share-grants.ts"; + + +// Transform / replay surface (CL-5872) +export { + createTransformConfig, + demoteGeneration, + listTransformConfigs, + promoteGeneration, + resolveGenerationSearchParams, + runTransform, + TransformConfigNotFoundError, + TransformPromoteError, + type GenerationSearchParams, + type TransformConfigRow, + type TransformRunRow, +} from "./services/transform.ts"; + +// Embed model registry (ensure vs activate) +export { + activateEmbedModel, + ensureEmbedModel, + resolveActiveEmbedTable, + resolveEmbedTableByModelKey, +} from "./core/embed-model-registry.ts"; + // Migrations export { runMemoryMigrations } from "./migrations.ts"; diff --git a/src/memory.ts b/src/memory.ts index e1d2f9f..d0d5e4d 100644 --- a/src/memory.ts +++ b/src/memory.ts @@ -26,6 +26,25 @@ import { listTimelineEvents, type TimelineEvent, } from "./services/timeline.ts"; +import { + createTransformConfig, + demoteGeneration, + listTransformConfigs, + promoteGeneration, + runTransform, + type TransformConfigRow, + type TransformRunRow, +} from "./services/transform.ts"; +import { + documentTag, + materializeShareGrants, + MEMORY_SHARE_CONDITION_REGISTRY, +} from "./services/share-grants.ts"; + +import { + isWritableGrantStore, +} from "./ports/writable-grant-store.ts"; +import type { TransformConfigParams, TransformScope } from "./core/schemas/transform.ts"; import { LIVE_TIMEOUT_MS, mergeLocalLiveV1, @@ -137,7 +156,8 @@ export type MemoryAddParams = MemoryIdentity & { */ accessTags?: string[]; /** - * Share sugar — only mints tags (tenant / peer owners / explicit tags). + * Share sugar — mints tags, and when the host grant store is writable, + * materializes peer grants on `memory.doc:` (CL-5873). */ share?: ShareSugar; attributes?: Record; @@ -182,6 +202,30 @@ export type Memory = { add(params: MemoryAddParams): Promise; list(params: MemoryListParams): Promise; close(): Promise; + /** + * Transform / replay surface (engine DocumentStore only). Present when the + * plane was built with engine config; absent on custom/fake stores. + * Calling through a stub that omits these is a TypeScript error; runtime + * throws MemoryError(501) only if a partial implementation is forced. + */ + createTransformConfig?(input: { + tenantId: string; + name: string; + params: TransformConfigParams; + }): Promise; + listTransformConfigs?(tenantId: string): Promise; + runTransform?(input: { + configId: string; + scope?: TransformScope; + }): Promise; + promoteGeneration?(input: { + tenantId: string; + generation: string; + }): Promise; + demoteGeneration?(input: { + tenantId: string; + generation: string; + }): Promise; }; export type { TimelineEvent }; @@ -220,9 +264,14 @@ export function resolveGrantConfig( options: Pick, ): GrantConfig | undefined { if (!options.grantStore) return undefined; + // Merge memoryShare evaluator under host keys so share grants with + // conditions are not fail-closed-skipped by @intx/authz. return { grantStore: options.grantStore, - conditionRegistry: options.conditionRegistry ?? {}, + conditionRegistry: { + ...MEMORY_SHARE_CONDITION_REGISTRY, + ...(options.conditionRegistry ?? {}), + }, }; } @@ -327,6 +376,8 @@ export function createMemory(options: MemoryOptions = {}): Memory { ...(grantStore !== undefined ? { grantStore } : {}), ...(conditionRegistry !== undefined ? { conditionRegistry } : {}), }); + + let transformDeps: EngineTransformDeps | undefined; const store = documentStore ?? (() => { @@ -336,8 +387,11 @@ export function createMemory(options: MemoryOptions = {}): Memory { "config is required when documentStore is not provided", ); } - return createEngineDocumentStore(config); + const engine = createEngineDocumentStore(config); + transformDeps = engine.deps; + return engine.store; })(); + return createPlaneFromStore( store, grants, @@ -345,6 +399,7 @@ export function createMemory(options: MemoryOptions = {}): Memory { ...(textExtractor ? { textExtractor } : {}), ...(sources ? { sources } : {}), }, + transformDeps, ); } @@ -507,6 +562,7 @@ function createPlaneFromStore( store: DocumentStore, grants: GrantConfig | undefined, options: MemoryOptions, + transformDeps?: EngineTransformDeps, ): Memory { async function searchMerged( params: MemorySearchParams, @@ -628,7 +684,7 @@ function createPlaneFromStore( params.externalRef ?? `memory:${params.tenantId}:${crypto.randomUUID()}`; - return store.add({ + const result = await store.add({ tenantId: params.tenantId, principalId: params.principalId, title, @@ -641,6 +697,38 @@ function createPlaneFromStore( ...(params.adapter !== undefined ? { adapter: params.adapter } : {}), ...(params.kind !== undefined ? { kind: params.kind } : {}), }); + + // Share materialization (CL-5873): stamp document-scoped tag + write + // peer grants when the host store is writable. Tag mint alone is not + // enough for peers without host bootstrap grants on owner tags. + const peers = params.share?.principals; + if (peers && peers.length > 0) { + const docTag = documentTag(result.documentId); + if (store.appendAccessTags) { + await store.appendAccessTags(result.documentId, [docTag]); + } else { + log.warn( + "memory.add: share.principals set but DocumentStore has no appendAccessTags; peer grants may not match", + { documentId: result.documentId }, + ); + } + if (isWritableGrantStore(grants?.grantStore)) { + await materializeShareGrants(grants.grantStore, { + tenantId: params.tenantId, + sharedByPrincipalId: params.principalId, + documentId: result.documentId, + sourceVersionId: result.documentId, + share: params.share ?? {}, + }); + } else { + log.warn( + "memory.add: share.principals set without WritableGrantStore; tags only (peers need host grants)", + { documentId: result.documentId }, + ); + } + } + + return result; }, async list(params) { @@ -659,17 +747,76 @@ function createPlaneFromStore( async close() { await store.close(); }, + + async createTransformConfig(input) { + if (!transformDeps) { + throw new MemoryError( + 501, + "transform APIs require the engine DocumentStore", + ); + } + return createTransformConfig({ db: transformDeps.db }, input); + }, + + async listTransformConfigs(tenantId) { + if (!transformDeps) { + throw new MemoryError( + 501, + "transform APIs require the engine DocumentStore", + ); + } + return listTransformConfigs({ db: transformDeps.db }, tenantId); + }, + + async runTransform(input) { + if (!transformDeps) { + throw new MemoryError( + 501, + "transform APIs require the engine DocumentStore", + ); + } + return runTransform(transformDeps, input); + }, + + async promoteGeneration(input) { + if (!transformDeps) { + throw new MemoryError( + 501, + "transform APIs require the engine DocumentStore", + ); + } + return promoteGeneration(transformDeps, input); + }, + + async demoteGeneration(input) { + if (!transformDeps) { + throw new MemoryError( + 501, + "transform APIs require the engine DocumentStore", + ); + } + return demoteGeneration(transformDeps, input); + }, }; return plane; } +type EngineTransformDeps = { + db: Db; + sql: RawSql; + config: EngineConfig; +}; + /** * Default DocumentStore: engine pgvector + hybrid search + timeline. * Owns construction-time rerank validation, FTS verification, and grant-tag * post-filter for document access. The plane never opens Postgres itself. */ -function createEngineDocumentStore(config: MemoryConfig): DocumentStore { +function createEngineDocumentStore(config: MemoryConfig): { + store: DocumentStore; + deps: EngineTransformDeps; +} { // Catch a chunk-size / reranker-limit mismatch at construction time, rather // than silently on every find once the reranker starts rejecting batches. // Throws instead of warning: a mismatch means every rerank call for this @@ -812,80 +959,100 @@ function createEngineDocumentStore(config: MemoryConfig): DocumentStore { } return { - async add(params) { - await ensureVerified(); + store: { + async add(params) { + await ensureVerified(); - const adapter = params.adapter ?? "http"; - const externalRef = - params.externalRef ?? - `memory:${params.tenantId}:${crypto.randomUUID()}`; - const accessTags = params.accessTags ?? [ownerTag(params.principalId)]; + const adapter = params.adapter ?? "http"; + const externalRef = + params.externalRef ?? + `memory:${params.tenantId}:${crypto.randomUUID()}`; + const accessTags = params.accessTags ?? [ownerTag(params.principalId)]; - const captureResult = await captureDocument(deps, { - tenantId: params.tenantId, - adapter, - occurredAt: new Date().toISOString(), - document: { - kind: params.kind ?? "note", - title: params.title, - externalRef, - accessTags, - entityHints: [], - chunks: [{ ordinal: 0, text: params.text }], - actor: { kind: "human", principalId: params.principalId }, - contentHash: "", // recomputed canonically in adapt-and-plan - ...(params.attributes !== undefined - ? { attributes: params.attributes } - : {}), - }, - }); - return { documentId: captureResult.documentId }; - }, + const captureResult = await captureDocument(deps, { + tenantId: params.tenantId, + adapter, + occurredAt: new Date().toISOString(), + document: { + kind: params.kind ?? "note", + title: params.title, + externalRef, + accessTags, + entityHints: [], + chunks: [{ ordinal: 0, text: params.text }], + actor: { kind: "human", principalId: params.principalId }, + contentHash: "", // recomputed canonically in adapt-and-plan + ...(params.attributes !== undefined + ? { attributes: params.attributes } + : {}), + }, + }); + return { documentId: captureResult.documentId }; + }, - async search(params) { - const result = await retrieve({ - tenantId: params.tenantId, - principalId: params.principalId, - query: params.query, - ...(params.limit !== undefined ? { k: params.limit } : {}), - ...(params.kinds !== undefined ? { kinds: params.kinds } : {}), - ...(params.entityIds !== undefined - ? { entityIds: params.entityIds } - : {}), - ...(params.grants !== undefined ? { grants: params.grants } : {}), - ...(params.conditionRegistry !== undefined - ? { conditionRegistry: params.conditionRegistry } - : {}), - }); - const items = hitsToSearchItems(result.hits); - if (params.includeEvidence) { + async appendAccessTags(documentId, tags) { + if (tags.length === 0) return; + // Union into existing access_tags array (postgres text[]). + await sql` + UPDATE "memory"."document" + SET access_tags = ( + SELECT ARRAY( + SELECT DISTINCT t + FROM unnest( + COALESCE(access_tags, '{}'::text[]) || ${[...tags]}::text[] + ) AS t + ) + ) + WHERE id = ${documentId} + `; + }, + + async search(params) { + const result = await retrieve({ + tenantId: params.tenantId, + principalId: params.principalId, + query: params.query, + ...(params.limit !== undefined ? { k: params.limit } : {}), + ...(params.kinds !== undefined ? { kinds: params.kinds } : {}), + ...(params.entityIds !== undefined + ? { entityIds: params.entityIds } + : {}), + ...(params.grants !== undefined ? { grants: params.grants } : {}), + ...(params.conditionRegistry !== undefined + ? { conditionRegistry: params.conditionRegistry } + : {}), + }); + const items = hitsToSearchItems(result.hits); + if (params.includeEvidence) { + return { + items, + evidence: result.evidence, + ...(result.degraded ? { degraded: result.degraded } : {}), + }; + } return { items, - evidence: result.evidence, ...(result.degraded ? { degraded: result.degraded } : {}), }; - } - return { - items, - ...(result.degraded ? { degraded: result.degraded } : {}), - }; - }, + }, - async list(params) { - return listTimelineEvents({ - db, - tenantId: params.tenantId, - principalId: params.principalId, - ...(params.limit !== undefined ? { limit: params.limit } : {}), - ...(params.grants !== undefined ? { grants: params.grants } : {}), - ...(params.conditionRegistry !== undefined - ? { conditionRegistry: params.conditionRegistry } - : {}), - }); - }, + async list(params) { + return listTimelineEvents({ + db, + tenantId: params.tenantId, + principalId: params.principalId, + ...(params.limit !== undefined ? { limit: params.limit } : {}), + ...(params.grants !== undefined ? { grants: params.grants } : {}), + ...(params.conditionRegistry !== undefined + ? { conditionRegistry: params.conditionRegistry } + : {}), + }); + }, - async close() { - await sql.end({ timeout: 5 }); + async close() { + await sql.end({ timeout: 5 }); + }, }, + deps, }; } diff --git a/src/ports/fakes.ts b/src/ports/fakes.ts index 8d63414..1cdcb0f 100644 --- a/src/ports/fakes.ts +++ b/src/ports/fakes.ts @@ -89,6 +89,14 @@ export function createFakeDocumentStore(): DocumentStore { return { documentId }; }, + async appendAccessTags(documentId, tags) { + const row = docs.find((d) => d.documentId === documentId); + if (!row) return; + const set = new Set(row.accessTags); + for (const t of tags) set.add(t); + row.accessTags = [...set]; + }, + async search( params: DocumentStoreSearchParams, ): Promise { diff --git a/src/ports/types.ts b/src/ports/types.ts index 8c3f785..be1c301 100644 --- a/src/ports/types.ts +++ b/src/ports/types.ts @@ -101,6 +101,15 @@ export type DocumentStore = { search(params: DocumentStoreSearchParams): Promise; list(params: DocumentStoreListParams): Promise; close(): Promise; + /** + * Append access tags after insert (used by share materialization to stamp + * `memory.doc:` once the document id is known). Optional — stores that + * omit it leave peer share grants without a matching tag (fail-closed). + */ + appendAccessTags?( + documentId: string, + tags: readonly string[], + ): Promise; }; /** diff --git a/src/ports/writable-grant-store.ts b/src/ports/writable-grant-store.ts new file mode 100644 index 0000000..5d572bb --- /dev/null +++ b/src/ports/writable-grant-store.ts @@ -0,0 +1,51 @@ +/** + * Host grant store that can materialize share grants. + * + * `@intx/authz` `GrantStore` is read-only (`collectGrants`). Memory never + * owns the grant plane — hosts that want share-on-add to work pass a store + * implementing this write seam. + */ +import type { GrantRule, GrantStore } from "@intx/authz"; + +export type WritableGrantStore = GrantStore & { + /** + * Insert or replace a grant by id. Hosts map this onto their control-plane + * grant table. Memory only calls this for share materialization. + */ + putGrant(grant: GrantRule): Promise; +}; + +export function isWritableGrantStore( + store: GrantStore | undefined, +): store is WritableGrantStore { + return ( + store !== undefined && + typeof (store as WritableGrantStore).putGrant === "function" + ); +} + +/** + * In-memory writable store for tests. Collects by principalId like + * `createInMemoryGrantStore` (tenantId accepted, unused). + */ +export function createInMemoryWritableGrantStore( + initial: GrantRule[] = [], +): WritableGrantStore & { grants: GrantRule[] } { + const grants = [...initial]; + return { + grants, + async collectGrants(principalId: string, _tenantId?: string) { + const now = new Date(); + return grants.filter((g) => { + if (g.principalId !== principalId) return false; + if (g.expiresAt !== null && g.expiresAt <= now) return false; + return true; + }); + }, + async putGrant(grant: GrantRule) { + const idx = grants.findIndex((g) => g.id === grant.id); + if (idx >= 0) grants[idx] = grant; + else grants.push(grant); + }, + }; +} diff --git a/src/services/capture.ts b/src/services/capture.ts index 715215b..6fcecc7 100644 --- a/src/services/capture.ts +++ b/src/services/capture.ts @@ -26,7 +26,7 @@ import type { } from "../core/schemas/adapted-document.ts"; import type { MemoryEdgeHint } from "../core/schemas/entity-edge.ts"; import { createRawSqlClient } from "../core/embed-sql.ts"; -import { activateEmbedModel } from "../core/embed-model-registry.ts"; +import { activateEmbedModel, ensureEmbedModel } from "../core/embed-model-registry.ts"; import type { EmbedClientConfig } from "../core/embed-client.ts"; import { embedChunks, type EmbeddableChunk } from "../core/embed-worker.ts"; import { toEmbedClientConfig } from "../core/engine-client-config.ts"; @@ -516,35 +516,38 @@ export { toEmbedClientConfig }; // Embeds a version's freshly-inserted chunks and stores their vectors, after // the derivation transaction has already committed. Best-effort in the -// fullest sense: ANY failure here — including activateEmbedModel's dims-probe -// network call, not just embedChunks' own client-error/rejected-chunk cases — -// is caught, logged, and swallowed. The chunk rows are already durable, and a -// later re-embed pass can pick up anything left unembedded (mirrors -// embed-worker.ts's pending-chunk contract, just invoked eagerly here instead -// of by polling). Returns whether embedding degraded so the caller can surface -// it. Shared by the live /capture path and a replay — the only difference -// between them is which `EmbedClientConfig` is passed in. +// fullest sense: ANY failure here — including ensure/activateEmbedModel's +// dims-probe network call, not just embedChunks' own client-error/rejected- +// chunk cases — is caught, logged, and swallowed. The chunk rows are already +// durable, and a later re-embed pass can pick up anything left unembedded +// (mirrors embed-worker.ts's pending-chunk contract, just invoked eagerly +// here instead of by polling). Returns whether embedding degraded so the +// caller can surface it. +// +// `promoteActive` (default true): live capture activates the model so dense +// search targets it. Replay must pass false so ensureEmbedModel only creates +// the table without flipping the tenant's active embed model (CL-5872). async function embedInsertedChunksWithConfig( sql: RawSql, tenantId: string, chunks: EmbeddableChunk[], embedClientConfig: EmbedClientConfig, + opts: { promoteActive?: boolean } = {}, ): Promise<{ degraded: boolean }> { if (chunks.length === 0) return { degraded: false }; try { const client = createRawSqlClient(sql); + const promoteActive = opts.promoteActive !== false; - const activeTable = await activateEmbedModel( - client, - tenantId, - embedClientConfig, - ); + const table = promoteActive + ? await activateEmbedModel(client, tenantId, embedClientConfig) + : await ensureEmbedModel(client, tenantId, embedClientConfig); const result = await embedChunks( client, tenantId, - activeTable, + table, chunks, embedClientConfig, ); @@ -646,6 +649,8 @@ export async function deriveFromRawCapture( input.tenantId, txResult.insertedChunks, derivation.embed, + // Replay never flips the tenant's active embed model. + { promoteActive: false }, ); return { diff --git a/src/services/search.ts b/src/services/search.ts index 81eb300..d78978c 100644 --- a/src/services/search.ts +++ b/src/services/search.ts @@ -11,8 +11,11 @@ import { import { createRawSqlClient } from "../core/embed-sql.ts"; import { cosineDistanceExpr, + computeModelKey, EMBED_TABLE_NAME_PATTERN, resolveActiveEmbedTable, + resolveEmbedTableByModelKey, + type ActiveEmbedTable, } from "../core/embed-model-registry.ts"; import { embedTexts, type EmbedClientConfig } from "../core/embed-client.ts"; import { @@ -430,15 +433,11 @@ interface FetchDenseCandidatesArgs extends ChannelFilterFields { principalId: string | null; query: string; overfetchLimit: number; - // Defaults to 'live' — see fetchLexicalCandidates' generation note. NOTE: - // this filters the chunk/version join, not which per-model embedding - // TABLE is queried — resolveActiveEmbedTable picks the tenant's single - // most-recently-activated model regardless of generation, so a replay run - // that activates a DIFFERENT embed model than the live one currently uses - // becomes the tenant's active table for every generation's dense channel, - // including live's. Scoping activation itself per-generation is out of - // the replay pipeline's scope; callers should reuse the live embed model in a - // transform_config unless they intend that tradeoff. + // Defaults to 'live' — see fetchLexicalCandidates' generation note. + // Dense table resolution is generation-aware: live uses the tenant's + // active embed model; a replay generation uses the model_key implied by + // the embed client config passed in (from that run's transform_config), + // which may be only `ready` and must not require status='active'. generation?: string | undefined; } @@ -484,7 +483,20 @@ export async function fetchDenseCandidates( if (query === "") return null; const embedSqlClient = createRawSqlClient(rawSql); - const activeTable = await resolveActiveEmbedTable(embedSqlClient, tenantId); + let activeTable: ActiveEmbedTable | null; + if (generation === LIVE_GENERATION) { + activeTable = await resolveActiveEmbedTable(embedSqlClient, tenantId); + } else { + const modelKey = computeModelKey( + embedClientConfig.baseUrl, + embedClientConfig.modelId, + ); + activeTable = await resolveEmbedTableByModelKey( + embedSqlClient, + tenantId, + modelKey, + ); + } if (!activeTable) return null; if (!EMBED_TABLE_NAME_PATTERN.test(activeTable.tableName)) { @@ -603,20 +615,32 @@ export async function fetchDenseCandidates( })); } -// Vectors for the MMR diversity pass, pulled from the tenant's ACTIVE -// per-model embedding table (never a superseded or inactive model's -// table). `pgvector` returns its column as text; the text form (`[1,2,3]`) -// is valid JSON, so `JSON.parse` is the exact inverse of the -// `JSON.stringify` the capture/embed pipeline writes on ingest. +// Vectors for the MMR diversity pass, pulled from the generation-scoped +// embedding table (active model for live; model_key for a replay generation). async function fetchChunkVectors( rawSql: RawSql, tenantId: string, chunkIds: readonly string[], + embedClientConfig: EmbedClientConfig, + generation: string = LIVE_GENERATION, ): Promise> { if (chunkIds.length === 0) return new Map(); const embedSqlClient = createRawSqlClient(rawSql); - const activeTable = await resolveActiveEmbedTable(embedSqlClient, tenantId); + let activeTable: ActiveEmbedTable | null; + if (generation === LIVE_GENERATION) { + activeTable = await resolveActiveEmbedTable(embedSqlClient, tenantId); + } else { + const modelKey = computeModelKey( + embedClientConfig.baseUrl, + embedClientConfig.modelId, + ); + activeTable = await resolveEmbedTableByModelKey( + embedSqlClient, + tenantId, + modelKey, + ); + } if (!activeTable) return new Map(); if (!EMBED_TABLE_NAME_PATTERN.test(activeTable.tableName)) { @@ -760,7 +784,7 @@ export async function hybridSearch( const resolvedTuning = generation === LIVE_GENERATION ? null - : await resolveGenerationSearchParams(db, generation); + : await resolveGenerationSearchParams(db, generation, config.embed); const authorityWeight = resolvedTuning?.authorityWeight ?? AUTHORITY_WEIGHT; const recencyHalfLifeMs = @@ -788,7 +812,8 @@ export async function hybridSearch( generation, }); - const embedClientConfig = toEmbedClientConfig(config.embed); + const embedClientConfig = + resolvedTuning?.embed ?? toEmbedClientConfig(config.embed); const rerankConfig = resolvedTuning?.rerank ?? toRerankClientConfig(config.rerank); let denseRows: CandidateRow[] = []; @@ -904,6 +929,8 @@ export async function hybridSearch( rawSql, tenantId, boosted.map((b) => b.row.chunkId), + embedClientConfig, + generation, ); const mmrItems: MmrItem[] = boosted.map((b) => ({ diff --git a/src/services/share-grants.test.ts b/src/services/share-grants.test.ts new file mode 100644 index 0000000..57c9177 --- /dev/null +++ b/src/services/share-grants.test.ts @@ -0,0 +1,178 @@ +import { describe, expect, it } from "bun:test"; +import { authorize } from "@intx/authz"; + +import { + buildShareGrants, + documentTag, + materializeShareGrants, + MEMORY_SHARE_CONDITION_REGISTRY, + shareWidenReceipt, + splitAudienceWiden, +} from "./share-grants.ts"; +import { createInMemoryWritableGrantStore } from "../ports/writable-grant-store.ts"; +import { canAccessDocument } from "../grant-tags.ts"; + +describe("documentTag", () => { + it("scopes resource to the document id", () => { + expect(documentTag("kdoc_1")).toBe("memory.doc:kdoc_1"); + }); +}); + +describe("buildShareGrants", () => { + it("emits one allow/search grant per peer on the document tag", () => { + const grants = buildShareGrants({ + tenantId: "t1", + sharedByPrincipalId: "alice", + documentId: "kdoc_1", + sourceVersionId: "kver_1", + share: { principals: ["bob", "carol"] }, + }); + expect(grants).toHaveLength(2); + expect(grants.every((g) => g.resource === "memory.doc:kdoc_1")).toBe(true); + expect(grants.every((g) => g.action === "search" && g.effect === "allow")).toBe( + true, + ); + expect(grants.map((g) => g.principalId).sort()).toEqual(["bob", "carol"]); + expect(grants[0]?.conditions?.memoryShare).toEqual({ + sharedBy: "alice", + sourceVersionId: "kver_1", + documentId: "kdoc_1", + tenantId: "t1", + }); + expect(grants[0]?.origin).toBe("system"); + }); + + it("skips the sharer themselves and empty principals", () => { + const grants = buildShareGrants({ + tenantId: "t1", + sharedByPrincipalId: "alice", + documentId: "kdoc_1", + sourceVersionId: "kver_1", + share: { principals: ["alice", " ", "bob"] }, + }); + expect(grants).toHaveLength(1); + expect(grants[0]?.principalId).toBe("bob"); + }); + + it("returns empty when only tenant/tags sugar is set", () => { + const grants = buildShareGrants({ + tenantId: "t1", + sharedByPrincipalId: "alice", + documentId: "kdoc_1", + sourceVersionId: "kver_1", + share: { tenant: true, tags: ["memory.space:eng"] }, + }); + expect(grants).toHaveLength(0); + }); +}); + +describe("materializeShareGrants + canAccessDocument", () => { + it("peer can search after materialize; non-peer cannot", async () => { + const store = createInMemoryWritableGrantStore(); + await materializeShareGrants(store, { + tenantId: "t1", + sharedByPrincipalId: "alice", + documentId: "kdoc_1", + sourceVersionId: "kver_1", + share: { principals: ["bob"] }, + }); + + const tags = ["memory.owner:alice", documentTag("kdoc_1")]; + const registry = MEMORY_SHARE_CONDITION_REGISTRY; + + expect( + await canAccessDocument({ + grants: store, + tenantId: "t1", + principalId: "bob", + createdByPrincipalId: "alice", + accessTags: tags, + conditionRegistry: registry, + }), + ).toBe(true); + + expect( + await canAccessDocument({ + grants: store, + tenantId: "t1", + principalId: "eve", + createdByPrincipalId: "alice", + accessTags: tags, + conditionRegistry: registry, + }), + ).toBe(false); + + // Creator still allowed without a grant. + expect( + await canAccessDocument({ + grants: store, + tenantId: "t1", + principalId: "alice", + createdByPrincipalId: "alice", + accessTags: tags, + }), + ).toBe(true); + }); + + it("embargo: expired grant does not allow access", async () => { + const store = createInMemoryWritableGrantStore(); + await store.putGrant({ + id: "g_expired", + principalId: "bob", + resource: "memory.doc:kdoc_1", + action: "search", + effect: "allow", + origin: "system", + roleId: null, + expiresAt: new Date("2020-01-01T00:00:00Z"), + conditions: null, + }); + + const decision = await authorize( + store, + "bob", + "t1", + "memory.doc:kdoc_1", + "search", + ); + expect(decision.effect).toBe(null); + + expect( + await canAccessDocument({ + grants: store, + tenantId: "t1", + principalId: "bob", + createdByPrincipalId: "alice", + accessTags: [documentTag("kdoc_1")], + }), + ).toBe(false); + }); +}); + +describe("splitAudienceWiden", () => { + it("keeps source tags, flags new ones for approval", () => { + const { allowed, needsApproval } = splitAudienceWiden( + ["memory.owner:alice", "memory.space:eng"], + ["memory.owner:alice", "memory.space:eng", "memory.tenant:t1"], + ); + expect(allowed).toEqual(["memory.owner:alice", "memory.space:eng"]); + expect(needsApproval).toEqual(["memory.tenant:t1"]); + }); +}); + +describe("shareWidenReceipt", () => { + it("records approver, tags, and source version", () => { + const receipt = shareWidenReceipt({ + approvedBy: "alice", + tags: ["memory.tenant:t1"], + sourceVersionId: "kver_1", + approvedAt: new Date("2026-07-20T00:00:00Z"), + }); + expect(receipt).toEqual({ + approvedBy: "alice", + approvedAt: "2026-07-20T00:00:00.000Z", + tags: ["memory.tenant:t1"], + sourceVersionId: "kver_1", + }); + }); +}); diff --git a/src/services/share-grants.ts b/src/services/share-grants.ts new file mode 100644 index 0000000..2a88f31 --- /dev/null +++ b/src/services/share-grants.ts @@ -0,0 +1,161 @@ +/** + * Share materialization — tags alone are not grants (CL-5873). + * + * Locked decisions (Greybeard): + * - Approver default: **source owner** (not tenant admin). + * - Staging: **write-narrow-then-widen** (no pending_share status). + * - Ask-on-read: design-only in v1 (canAccessDocument still fail-closed on ask). + * + * Origin is constrained by `@intx/types` to system|role|creator|invoker — + * memory-share provenance lives in `conditions.memoryShare` (audit payload). + * Authz skips grants with non-null conditions unless a registry is provided; + * use `MEMORY_SHARE_CONDITION_REGISTRY` (merged automatically in resolveGrantConfig). + */ +import type { ConditionRegistry, GrantRule } from "@intx/authz"; +import { newId } from "../core/id.ts"; +import type { ShareSugar } from "../grant-tags.ts"; +import type { WritableGrantStore } from "../ports/writable-grant-store.ts"; + +/** Document-scoped resource tag for peer share grants. */ +export function documentTag(documentId: string): string { + return `memory.doc:${documentId}`; +} + +export const MEMORY_SHARE_CONDITION_KEY = "memoryShare"; + +/** + * Audit payload stored under conditions.memoryShare. + * The evaluator always returns true — this is provenance, not a gate. + */ +export type MemoryShareCondition = { + sharedBy: string; + sourceVersionId: string; + documentId: string; + tenantId: string; +}; + +/** + * Default registry so share grants with conditions are not fail-closed-skipped. + * Hosts may override the key; resolveGrantConfig merges host keys on top. + */ +export const MEMORY_SHARE_CONDITION_REGISTRY: ConditionRegistry = { + [MEMORY_SHARE_CONDITION_KEY]: () => true, +}; + +export type MaterializeShareGrantsInput = { + tenantId: string; + /** Principal who initiated the share (source owner / creator). */ + sharedByPrincipalId: string; + documentId: string; + /** Version that carried the share (for audit receipt). */ + sourceVersionId: string; + share: ShareSugar; +}; + +/** + * Build grant rules for peer principals on a document-scoped tag. + * + * - `share.principals`: one allow/search grant per peer on `memory.doc:`. + * - `share.tenant` / `share.tags`: do **not** auto-mint principal grants — + * those rely on host role/pattern grants already present on the tag. + * + * Returns rules only (does not write). Caller applies via WritableGrantStore. + */ +export function buildShareGrants( + input: MaterializeShareGrantsInput, +): GrantRule[] { + const peers = input.share.principals ?? []; + if (peers.length === 0) return []; + + const resource = documentTag(input.documentId); + const rules: GrantRule[] = []; + + for (const peer of peers) { + if (typeof peer !== "string" || peer.trim() === "") continue; + const principalId = peer.trim(); + // Never grant the owner to themselves via share — creator path already covers. + if (principalId === input.sharedByPrincipalId) continue; + + const sharePayload: MemoryShareCondition = { + sharedBy: input.sharedByPrincipalId, + sourceVersionId: input.sourceVersionId, + documentId: input.documentId, + tenantId: input.tenantId, + }; + + rules.push({ + id: newId("mgrt"), + principalId, + resource, + action: "search", + effect: "allow", + origin: "system", + roleId: null, + expiresAt: null, + // Single condition key so hosts only need one registry entry. + // Nested object carries audit provenance without extra evaluators. + conditions: { + [MEMORY_SHARE_CONDITION_KEY]: sharePayload, + }, + }); + } + + return rules; +} + +/** + * Write share grants to the host store. + */ +export async function materializeShareGrants( + store: WritableGrantStore, + input: MaterializeShareGrantsInput, +): Promise<{ written: number; grants: GrantRule[] }> { + const grants = buildShareGrants(input); + for (const grant of grants) { + await store.putGrant(grant); + } + return { written: grants.length, grants }; +} + +/** + * Split proposed access tags into those already covered by the source + * audience vs those that widen (need source-owner approval). + * + * Write-narrow-then-widen: caller writes with `allowed` only, then widens + * after approval by appending `needsApproval` tags + materializing grants. + */ +export function splitAudienceWiden( + sourceAccessTags: readonly string[], + proposedAccessTags: readonly string[], +): { allowed: string[]; needsApproval: string[] } { + const source = new Set(sourceAccessTags); + const allowed: string[] = []; + const needsApproval: string[] = []; + for (const tag of proposedAccessTags) { + if (source.has(tag)) allowed.push(tag); + else needsApproval.push(tag); + } + return { allowed, needsApproval }; +} + +/** Receipt shape stored on version attributes after widen approval. */ +export type ShareWidenReceipt = { + approvedBy: string; + approvedAt: string; + tags: string[]; + sourceVersionId: string; +}; + +export function shareWidenReceipt(params: { + approvedBy: string; + tags: readonly string[]; + sourceVersionId: string; + approvedAt?: Date; +}): ShareWidenReceipt { + return { + approvedBy: params.approvedBy, + approvedAt: (params.approvedAt ?? new Date()).toISOString(), + tags: [...params.tags], + sourceVersionId: params.sourceVersionId, + }; +} diff --git a/src/services/transform.ts b/src/services/transform.ts index 39fea77..2a752ff 100644 --- a/src/services/transform.ts +++ b/src/services/transform.ts @@ -4,7 +4,12 @@ import type { Db, RawSql } from "../db/client.ts"; import type { EngineConfig } from "../config.ts"; import { newId } from "../core/id.ts"; import { formatCaughtError, log } from "../log.ts"; -import { rawCapture, transformConfig, transformRun } from "../db/schema.ts"; +import { + memoryVersion, + rawCapture, + transformConfig, + transformRun, +} from "../db/schema.ts"; import { TransformConfigParamsSchema, type TransformConfigParams, @@ -17,6 +22,9 @@ import type { Chunker } from "../core/chunk/types.ts"; import { EmbedClientConfigSchema, type EmbedClientConfig } from "../core/embed-client.ts"; import type { RerankClientConfig } from "../core/rerank-client.ts"; import { deriveFromRawCapture, type CaptureInput } from "./capture.ts"; +import { LIVE_GENERATION } from "../core/generation.ts"; +import { activateEmbedModel } from "../core/embed-model-registry.ts"; +import { createRawSqlClient } from "../core/embed-sql.ts"; export class TransformConfigNotFoundError extends Error { constructor(configId: string) { @@ -46,6 +54,8 @@ export interface TransformRunRow { error: string | null; createdAt: Date; completedAt: Date | null; + archivedLiveGeneration: string | null; + promotedAt: Date | null; } // Parses the jsonb `params` column at this trust boundary — the row was @@ -161,6 +171,8 @@ async function loadTransformRun( error: row.error, createdAt: row.createdAt, completedAt: row.completedAt, + archivedLiveGeneration: row.archivedLiveGeneration ?? null, + promotedAt: row.promotedAt ?? null, }; } @@ -225,6 +237,8 @@ export interface GenerationSearchParams { mmrLambda: number | undefined; overfetch: number | undefined; rerank: RerankClientConfig | undefined; + /** Fully-resolved embed client for this generation's transform_config. */ + embed: EmbedClientConfig | undefined; } // Resolves a search-time `generation` (a transform_run id, per the 1:1 @@ -232,9 +246,13 @@ export interface GenerationSearchParams { // Returns `null` when the generation isn't a known replay run (including // 'live', which the caller should never even ask this for) — hybridSearch // falls back to its own engine defaults for every field in that case. +// +// `engineEmbed` is required to fully resolve a partial transform embed +// override (same merge rules as runTransform). export async function resolveGenerationSearchParams( db: Db, generation: string, + engineEmbed?: EngineConfig["embed"], ): Promise { const runRows = await db .select({ configId: transformRun.configId }) @@ -253,12 +271,17 @@ export async function resolveGenerationSearchParams( if (!configRow) return null; const params = parseConfigParams(configRow.params); + const embed = + engineEmbed !== undefined + ? buildEmbedClientConfig(params.embed, engineEmbed) + : undefined; return { authorityWeight: params.authorityWeight, recencyHalfLifeDays: params.recencyHalfLifeDays, mmrLambda: params.mmrLambda, overfetch: params.overfetch, rerank: buildRerankClientConfig(params.rerank), + embed, }; } @@ -442,3 +465,170 @@ export async function runTransform( return loadTransformRun(deps.db, runId); } + +export class TransformPromoteError extends Error { + constructor(message: string) { + super(message); + this.name = "TransformPromoteError"; + } +} + +/** + * Promote a completed staged generation to live. + * + * 1. Move current `live` versions to a unique archive generation (prior intact). + * 2. Move staged generation versions onto `live`. + * 3. Activate the run's embed model so dense search targets the promoted corpus. + * 4. Record archive tag + promoted_at on the run for demote. + * + * Does not delete versions. Demote reverses the generation swap and re-activates + * the prior live embed model when still registered. + */ +export async function promoteGeneration( + deps: { db: Db; sql: RawSql; config: EngineConfig }, + input: { tenantId: string; generation: string }, +): Promise { + if (input.generation === LIVE_GENERATION) { + throw new TransformPromoteError("cannot promote the live generation onto itself"); + } + + const runRows = await deps.db + .select() + .from(transformRun) + .where( + and( + eq(transformRun.generation, input.generation), + eq(transformRun.tenantId, input.tenantId), + ), + ) + .limit(1); + const run = runRows[0]; + if (!run) { + throw new TransformPromoteError( + `no transform_run for generation ${input.generation}`, + ); + } + if (run.promotedAt) { + throw new TransformPromoteError( + `generation ${input.generation} is already promoted`, + ); + } + if (run.status === "running") { + throw new TransformPromoteError( + `generation ${input.generation} is still running`, + ); + } + + const configRow = await loadTransformConfig(deps.db, run.configId); + const embed = buildEmbedClientConfig(configRow.params.embed, deps.config.embed); + const archiveGen = `archive_${run.id}_${Date.now()}`; + + await deps.db.transaction(async (tx) => { + // 1) archive current live + await tx + .update(memoryVersion) + .set({ generation: archiveGen }) + .where( + and( + eq(memoryVersion.tenantId, input.tenantId), + eq(memoryVersion.generation, LIVE_GENERATION), + ), + ); + // 2) promote staged → live + await tx + .update(memoryVersion) + .set({ generation: LIVE_GENERATION }) + .where( + and( + eq(memoryVersion.tenantId, input.tenantId), + eq(memoryVersion.generation, input.generation), + ), + ); + // 3) bookkeeping — generation column stays the original run id for lookup; + // versions now live under 'live'. Search by generation=runId after + // promote finds nothing (expected); demote restores. + await tx + .update(transformRun) + .set({ + archivedLiveGeneration: archiveGen, + promotedAt: new Date(), + }) + .where(eq(transformRun.id, run.id)); + }); + + // Activate embed outside the txn — DDL/network, not version rows. + const client = createRawSqlClient(deps.sql); + await activateEmbedModel(client, input.tenantId, embed); + + return loadTransformRun(deps.db, run.id); +} + +/** + * Demote a previously promoted generation: swap archive back to live and + * move the demoted live corpus back onto the run's generation tag. + */ +export async function demoteGeneration( + deps: { db: Db; sql: RawSql; config: EngineConfig }, + input: { tenantId: string; generation: string }, +): Promise { + const runRows = await deps.db + .select() + .from(transformRun) + .where( + and( + eq(transformRun.generation, input.generation), + eq(transformRun.tenantId, input.tenantId), + ), + ) + .limit(1); + const run = runRows[0]; + if (!run) { + throw new TransformPromoteError( + `no transform_run for generation ${input.generation}`, + ); + } + if (!run.promotedAt || !run.archivedLiveGeneration) { + throw new TransformPromoteError( + `generation ${input.generation} is not currently promoted`, + ); + } + + const archiveGen = run.archivedLiveGeneration; + + await deps.db.transaction(async (tx) => { + // live (promoted) → back to run generation + await tx + .update(memoryVersion) + .set({ generation: input.generation }) + .where( + and( + eq(memoryVersion.tenantId, input.tenantId), + eq(memoryVersion.generation, LIVE_GENERATION), + ), + ); + // archive → live + await tx + .update(memoryVersion) + .set({ generation: LIVE_GENERATION }) + .where( + and( + eq(memoryVersion.tenantId, input.tenantId), + eq(memoryVersion.generation, archiveGen), + ), + ); + await tx + .update(transformRun) + .set({ + archivedLiveGeneration: null, + promotedAt: null, + }) + .where(eq(transformRun.id, run.id)); + }); + + // Note: demote does not auto-activate a prior embed model — the host may + // re-activate via a subsequent live capture or explicit promote of another run. + // Live dense search continues against whichever model is currently active; + // vectors for restored live versions remain in their original embed tables. + + return loadTransformRun(deps.db, run.id); +} From fb0fe3d9090a7c2ff7d8f869a6fe7a3600c01eea Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 10 Aug 2026 09:32:39 -0700 Subject: [PATCH 04/19] Restore prior embed model on demote and return version ids from add Promote only completed runs, activate the staged embed model before swapping generation tags, and record the pre-promote model key so demote can restore dense search fail-closed. Add returns versionId so share grants cite the written version rather than the document id. --- IMPLEMENTATION.md | 22 ++-- migrations/0005_transform_promote.sql | 3 + src/core/embed-model-registry.test.ts | 48 +++++++++ src/core/embed-model-registry.ts | 37 +++++++ src/db/schema.ts | 2 + src/index.ts | 3 + src/memory.test.ts | 4 +- src/memory.ts | 9 +- src/ports/fakes.ts | 2 +- src/ports/types.ts | 8 +- src/routes/add.ts | 5 +- src/routes/routes.test.ts | 5 +- src/services/transform.test.ts | 13 ++- src/services/transform.ts | 144 +++++++++++++++++--------- src/tools/add.ts | 2 +- src/tools/tools.test.ts | 8 +- 16 files changed, 241 insertions(+), 74 deletions(-) diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md index ffcd2f1..6e85970 100644 --- a/IMPLEMENTATION.md +++ b/IMPLEMENTATION.md @@ -252,8 +252,10 @@ Two write paths (CL-5872): `resolveEmbedTableByModelKey` against the owning transform config's embed `model_key` — ready or active — so staged generations never steal live. -Optional `archived_live_generation` on `transform_run` records which generation -held live rows before promote (for demote/rollback). +Optional `archived_live_generation` and `archived_live_model_key` on +`transform_run` record which generation and dense model held live rows before +promote (for demote/rollback of both corpus and dense search). + ### Dynamic per-model vector tables: `memory_embedding_` Not in `db/schema.ts` (no fixed shape — dimensionality varies by model) and @@ -467,7 +469,7 @@ search); otherwise it throws `MemorySearchInputError` (400). entity edges; `toHit` builds the wire `SearchHit` (citation/open-target resolution via `openTarget`, mapping known adapters — `artifact`, `task`, `workflow_run`, `mail` — to a deep-linkable `{type, id}`, else a generic - `{type: "knowledge", id: documentId}`). + `{type: "memory", id: documentId}`). 11. **Evidence** — `deriveHybridEvidence`: `"none"` if zero hits; `"weak"` if the lexical channel contributed zero rows (a dense-only result never reports `"strong"`); otherwise `deriveEvidence` on the lexical rows — @@ -511,12 +513,14 @@ writes to. A `transform_config` is a named/versioned recipe **Promote / demote (staged cutover):** -- `promoteGeneration` — swaps `status` so the staged generation becomes - `active` under `generation = 'live'` semantics for search (prior live is - archived; `archived_live_generation` records the previous id for rollback). - Optionally activates the generation's embed model so live dense matches - the promoted embeddings. -- `demoteGeneration` — restores the archived live generation. +- `promoteGeneration` — requires `status = 'completed'`. Snapshots the + pre-promote active `model_key`, activates the staged embed model, then + swaps generation tags (`live` → archive, staged → `live`). Records + `archived_live_generation` + `archived_live_model_key` for demote. If the + version swap fails after activate, re-activates the prior model_key. +- `demoteGeneration` — re-activates `archived_live_model_key` (fail-closed if + the registry row is gone), then restores archive → live and staged corpus + back onto the run generation. Plane methods (engine DocumentStore only): `createTransformConfig`, `listTransformConfigs`, `runTransform`, `promoteGeneration`, diff --git a/migrations/0005_transform_promote.sql b/migrations/0005_transform_promote.sql index 40ffcaa..2631932 100644 --- a/migrations/0005_transform_promote.sql +++ b/migrations/0005_transform_promote.sql @@ -1,7 +1,10 @@ -- Promote / demote bookkeeping for staged transform generations (CL-5872). -- archived_live_generation holds the generation tag assigned to the prior -- live corpus during promote, so demote can swap back without data loss. +-- archived_live_model_key holds the pre-promote active embed model_key so +-- demote can restore dense search to the table that holds restored vectors. ALTER TABLE "memory"."transform_run" ADD COLUMN IF NOT EXISTS "archived_live_generation" text, + ADD COLUMN IF NOT EXISTS "archived_live_model_key" text, ADD COLUMN IF NOT EXISTS "promoted_at" timestamp with time zone; diff --git a/src/core/embed-model-registry.test.ts b/src/core/embed-model-registry.test.ts index 30ab10e..5869962 100644 --- a/src/core/embed-model-registry.test.ts +++ b/src/core/embed-model-registry.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, mock } from "bun:test"; import type { EmbedClientConfig } from "./embed-client.ts"; import { activateEmbedModel, + activateEmbedModelByKey, computeModelKey, cosineDistanceExpr, DimsOutOfBoundsError, @@ -316,6 +317,53 @@ describe("activateEmbedModel (split)", () => { expect(updateQuery).toBeDefined(); expect(updateQuery?.sql).toContain("updated_at = now()"); }); + + it("ensure then activate does not leave only-ready when activate follows", async () => { + const { client, queries } = createMockClient(); + await ensureEmbedModel(client, "tenant-1", baseConfig, fixtureFetch(768)); + const ensureQueries = queries.length; + await activateEmbedModel(client, "tenant-1", baseConfig, fixtureFetch(768)); + // activate always issues the status UPDATE after ensure work + expect( + queries.slice(ensureQueries).some((q) => q.sql.includes("SET status = 'active'")), + ).toBe(true); + }); +}); + +describe("activateEmbedModelByKey", () => { + it("issues UPDATE by model_key without probing the embed endpoint", async () => { + const modelKey = "abcdef0123456789"; + const queries: Array<{ sql: string; params: readonly unknown[] }> = []; + const client: EmbedRegistrySqlClient = { + query: (sql, params) => { + queries.push({ sql, params }); + return Promise.resolve([ + { model_key: modelKey, model_id: "text-embed-3", dims: 768 }, + ]); + }, + }; + const result = await activateEmbedModelByKey(client, "tenant-1", modelKey); + expect(result.modelKey).toBe(modelKey); + expect(result.dims).toBe(768); + expect(queries[0]?.sql).toContain("SET status = 'active'"); + expect(queries[0]?.params).toEqual(["tenant-1", modelKey]); + }); + + it("throws when the registry row is missing", async () => { + const client: EmbedRegistrySqlClient = { + query: () => Promise.resolve([]), + }; + await expect( + activateEmbedModelByKey(client, "tenant-1", "abcdef0123456789"), + ).rejects.toThrow(/no embed_model row/); + }); + + it("rejects an invalid model_key format", async () => { + const client: EmbedRegistrySqlClient = { query: () => Promise.resolve([]) }; + await expect( + activateEmbedModelByKey(client, "tenant-1", "not-a-key"), + ).rejects.toThrow(/invalid modelKey/); + }); }); describe("resolveEmbedTableByModelKey", () => { diff --git a/src/core/embed-model-registry.ts b/src/core/embed-model-registry.ts index e3079ca..2278a54 100644 --- a/src/core/embed-model-registry.ts +++ b/src/core/embed-model-registry.ts @@ -254,6 +254,43 @@ export async function resolveActiveEmbedTable( }; } +/** + * Promote an already-registered embed model to the tenant's active dense + * target by `model_key` only (no embed endpoint probe). Used by demote to + * restore the pre-promote live model without needing baseUrl/modelId. + * + * Throws if the registry row is missing — demote must not silently leave + * dense search on the promoted model. + */ +export async function activateEmbedModelByKey( + client: EmbedRegistrySqlClient, + tenantId: string, + modelKey: string, +): Promise { + if (!/^[a-f0-9]{16}$/.test(modelKey)) { + throw new Error(`activateEmbedModelByKey: invalid modelKey "${modelKey}"`); + } + const rows = await client.query( + `UPDATE "memory"."embed_model" + SET status = 'active', updated_at = now() + WHERE tenant_id = $1 AND model_key = $2 + RETURNING model_key, model_id, dims`, + [tenantId, modelKey], + ); + const row = rows[0]; + if (!row) { + throw new Error( + `activateEmbedModelByKey: no embed_model row for tenant=${tenantId} model_key=${modelKey}`, + ); + } + return { + tableName: embeddingTableName(modelKey), + dims: row.dims as number, + modelId: row.model_id as string, + modelKey, + }; +} + /** * Resolve any registered embed table by model_key (active or ready). * Used when searching a non-live generation whose transform_config embeds diff --git a/src/db/schema.ts b/src/db/schema.ts index 8c20b34..5337c67 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -236,6 +236,8 @@ export const transformRun = memorySchema.table( completedAt: timestamp("completed_at"), /** Generation tag assigned to the prior live corpus on promote (for demote). */ archivedLiveGeneration: text("archived_live_generation"), + /** Pre-promote active embed model_key (for demote dense restore). */ + archivedLiveModelKey: text("archived_live_model_key"), promotedAt: timestamp("promoted_at"), }, (t) => [ diff --git a/src/index.ts b/src/index.ts index a5dec49..596104c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -60,6 +60,7 @@ export { export type { DocumentStore, DocumentStoreAddParams, + DocumentStoreAddResult, DocumentStoreSearchItem, DocumentStoreSearchParams, DocumentStoreSearchResult, @@ -113,11 +114,13 @@ export { // Embed model registry (ensure vs activate) export { activateEmbedModel, + activateEmbedModelByKey, ensureEmbedModel, resolveActiveEmbedTable, resolveEmbedTableByModelKey, } from "./core/embed-model-registry.ts"; + // Migrations export { runMemoryMigrations } from "./migrations.ts"; diff --git a/src/memory.test.ts b/src/memory.test.ts index 11fc1c4..562e0ad 100644 --- a/src/memory.test.ts +++ b/src/memory.test.ts @@ -514,7 +514,7 @@ async function freshPlane(opts?: { principalId: PRINCIPAL, content: { title: "T", text: "body" }, }); - expect(result).toEqual({ documentId: "kdoc_captured" }); + expect(result).toEqual({ documentId: "kdoc_captured", versionId: "kver_1" }); await plane.close(); }); @@ -534,7 +534,7 @@ async function freshPlane(opts?: { principalId: PRINCIPAL, content: { title: "T", text: "body" }, }); - expect(result).toEqual({ documentId: "kdoc_noop" }); + expect(result).toEqual({ documentId: "kdoc_noop", versionId: "kver_1" }); await plane.close(); }); diff --git a/src/memory.ts b/src/memory.ts index d0d5e4d..a24b004 100644 --- a/src/memory.ts +++ b/src/memory.ts @@ -163,7 +163,7 @@ export type MemoryAddParams = MemoryIdentity & { attributes?: Record; }; -export type MemoryAddResult = { documentId: string }; +export type MemoryAddResult = { documentId: string; versionId: string }; export type SearchItem = { documentId: string; @@ -717,7 +717,7 @@ function createPlaneFromStore( tenantId: params.tenantId, sharedByPrincipalId: params.principalId, documentId: result.documentId, - sourceVersionId: result.documentId, + sourceVersionId: result.versionId, share: params.share ?? {}, }); } else { @@ -987,7 +987,10 @@ function createEngineDocumentStore(config: MemoryConfig): { : {}), }, }); - return { documentId: captureResult.documentId }; + return { + documentId: captureResult.documentId, + versionId: captureResult.versionId, + }; }, async appendAccessTags(documentId, tags) { diff --git a/src/ports/fakes.ts b/src/ports/fakes.ts index 1cdcb0f..e1ecf0e 100644 --- a/src/ports/fakes.ts +++ b/src/ports/fakes.ts @@ -86,7 +86,7 @@ export function createFakeDocumentStore(): DocumentStore { row.externalRef = params.externalRef; } docs.push(row); - return { documentId }; + return { documentId, versionId: `fake_ver_${seq}` }; }, async appendAccessTags(documentId, tags) { diff --git a/src/ports/types.ts b/src/ports/types.ts index be1c301..651003e 100644 --- a/src/ports/types.ts +++ b/src/ports/types.ts @@ -96,8 +96,14 @@ export type DocumentStoreListEvent = { * to replace local Postgres entirely — this is the only product path for * swapping backends. */ +export type DocumentStoreAddResult = { + documentId: string; + /** Active version written (or existing active on noop). */ + versionId: string; +}; + export type DocumentStore = { - add(params: DocumentStoreAddParams): Promise<{ documentId: string }>; + add(params: DocumentStoreAddParams): Promise; search(params: DocumentStoreSearchParams): Promise; list(params: DocumentStoreListParams): Promise; close(): Promise; diff --git a/src/routes/add.ts b/src/routes/add.ts index 7e5a979..fc99b50 100644 --- a/src/routes/add.ts +++ b/src/routes/add.ts @@ -13,6 +13,7 @@ import { caller, grantGuard, requirePrincipal } from "./deps.ts"; const AddResponse = type({ documentId: "string", + versionId: "string", }); export function mountAddRoute(app: Hono, deps: RouteDeps): void { @@ -57,14 +58,14 @@ export function mountAddRoute(app: Hono, deps: RouteDeps): void { } try { - const { documentId } = await deps.memory.add({ + const result = await deps.memory.add({ content: { title, text }, tenantId: scopeId, principalId: subjectId, ...(accessTags !== undefined ? { accessTags } : {}), ...(share !== undefined ? { share } : {}), }); - return c.json({ documentId }); + return c.json({ documentId: result.documentId, versionId: result.versionId }); } catch (err) { if (err instanceof MemoryError) { return c.json( diff --git a/src/routes/routes.test.ts b/src/routes/routes.test.ts index 4607286..70bd434 100644 --- a/src/routes/routes.test.ts +++ b/src/routes/routes.test.ts @@ -50,7 +50,7 @@ function stubPlane(opts?: { tenantId: p.tenantId, principalId: p.principalId, }); - return { documentId: "doc-stub" }; + return { documentId: "doc-stub", versionId: "ver-stub" }; }, list: async (p) => { return catalog @@ -168,8 +168,9 @@ describe("memory HTTP routes", () => { jsonPost({ title: "t", text: "body" }), ); expect(res.status).toBe(200); - const body = (await res.json()) as { documentId: string }; + const body = (await res.json()) as { documentId: string; versionId: string }; expect(body.documentId).toBe("doc-stub"); + expect(body.versionId).toBe("ver-stub"); expect(added).toEqual([ { title: "t", tenantId: TENANT, principalId: PRINCIPAL }, ]); diff --git a/src/services/transform.test.ts b/src/services/transform.test.ts index c640faf..d0453ea 100644 --- a/src/services/transform.test.ts +++ b/src/services/transform.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from "bun:test"; import { type } from "arktype"; -import { buildRerankClientConfig } from "./transform.ts"; +import { + buildRerankClientConfig, + isPromotableRunStatus, +} from "./transform.ts"; import { TransformConfigParamsSchema } from "../core/schemas/transform.ts"; describe("buildRerankClientConfig", () => { @@ -31,6 +34,14 @@ describe("buildRerankClientConfig", () => { }); }); +describe("isPromotableRunStatus", () => { + it("allows only completed runs", () => { + expect(isPromotableRunStatus("completed")).toBe(true); + expect(isPromotableRunStatus("running")).toBe(false); + expect(isPromotableRunStatus("failed")).toBe(false); + }); +}); + describe("TransformConfigParamsSchema", () => { it("accepts a fully-specified params object", () => { const parsed = TransformConfigParamsSchema({ diff --git a/src/services/transform.ts b/src/services/transform.ts index 2a752ff..cba261c 100644 --- a/src/services/transform.ts +++ b/src/services/transform.ts @@ -23,7 +23,7 @@ import { EmbedClientConfigSchema, type EmbedClientConfig } from "../core/embed-c import type { RerankClientConfig } from "../core/rerank-client.ts"; import { deriveFromRawCapture, type CaptureInput } from "./capture.ts"; import { LIVE_GENERATION } from "../core/generation.ts"; -import { activateEmbedModel } from "../core/embed-model-registry.ts"; +import { activateEmbedModel, activateEmbedModelByKey, resolveActiveEmbedTable } from "../core/embed-model-registry.ts"; import { createRawSqlClient } from "../core/embed-sql.ts"; export class TransformConfigNotFoundError extends Error { @@ -55,6 +55,7 @@ export interface TransformRunRow { createdAt: Date; completedAt: Date | null; archivedLiveGeneration: string | null; + archivedLiveModelKey: string | null; promotedAt: Date | null; } @@ -172,6 +173,7 @@ async function loadTransformRun( createdAt: row.createdAt, completedAt: row.completedAt, archivedLiveGeneration: row.archivedLiveGeneration ?? null, + archivedLiveModelKey: row.archivedLiveModelKey ?? null, promotedAt: row.promotedAt ?? null, }; } @@ -473,16 +475,28 @@ export class TransformPromoteError extends Error { } } +/** + * True when a transform_run may be promoted to live. Exported for unit tests. + * Only completed runs are promotable — failed/running leave a partial corpus. + */ +export function isPromotableRunStatus( + status: TransformRunRow["status"], +): status is "completed" { + return status === "completed"; +} + /** * Promote a completed staged generation to live. * - * 1. Move current `live` versions to a unique archive generation (prior intact). - * 2. Move staged generation versions onto `live`. - * 3. Activate the run's embed model so dense search targets the promoted corpus. - * 4. Record archive tag + promoted_at on the run for demote. + * 1. Snapshot the current active embed model_key (for demote restore). + * 2. Activate the run's embed model first so a failed activate leaves versions + * untouched (brief dense mismatch window is preferred over committed corpus + * with no matching dense table). + * 3. Swap generations: live → archive tag, staged → live. + * 4. Record archive tag + prior model_key + promoted_at for demote. * * Does not delete versions. Demote reverses the generation swap and re-activates - * the prior live embed model when still registered. + * the prior live embed model when recorded. */ export async function promoteGeneration( deps: { db: Db; sql: RawSql; config: EngineConfig }, @@ -513,59 +527,83 @@ export async function promoteGeneration( `generation ${input.generation} is already promoted`, ); } - if (run.status === "running") { + if (!isPromotableRunStatus(run.status as TransformRunRow["status"])) { throw new TransformPromoteError( - `generation ${input.generation} is still running`, + `generation ${input.generation} is not completed (status=${run.status})`, ); } const configRow = await loadTransformConfig(deps.db, run.configId); const embed = buildEmbedClientConfig(configRow.params.embed, deps.config.embed); const archiveGen = `archive_${run.id}_${Date.now()}`; + const client = createRawSqlClient(deps.sql); - await deps.db.transaction(async (tx) => { - // 1) archive current live - await tx - .update(memoryVersion) - .set({ generation: archiveGen }) - .where( - and( - eq(memoryVersion.tenantId, input.tenantId), - eq(memoryVersion.generation, LIVE_GENERATION), - ), - ); - // 2) promote staged → live - await tx - .update(memoryVersion) - .set({ generation: LIVE_GENERATION }) - .where( - and( - eq(memoryVersion.tenantId, input.tenantId), - eq(memoryVersion.generation, input.generation), - ), - ); - // 3) bookkeeping — generation column stays the original run id for lookup; - // versions now live under 'live'. Search by generation=runId after - // promote finds nothing (expected); demote restores. - await tx - .update(transformRun) - .set({ - archivedLiveGeneration: archiveGen, - promotedAt: new Date(), - }) - .where(eq(transformRun.id, run.id)); - }); + // Snapshot prior active model before we flip dense search. + const priorActive = await resolveActiveEmbedTable(client, input.tenantId); + const priorModelKey = priorActive?.modelKey ?? null; - // Activate embed outside the txn — DDL/network, not version rows. - const client = createRawSqlClient(deps.sql); + // Activate staged embed first — if this fails, versions stay put. await activateEmbedModel(client, input.tenantId, embed); + try { + await deps.db.transaction(async (tx) => { + // 1) archive current live + await tx + .update(memoryVersion) + .set({ generation: archiveGen }) + .where( + and( + eq(memoryVersion.tenantId, input.tenantId), + eq(memoryVersion.generation, LIVE_GENERATION), + ), + ); + // 2) promote staged → live + await tx + .update(memoryVersion) + .set({ generation: LIVE_GENERATION }) + .where( + and( + eq(memoryVersion.tenantId, input.tenantId), + eq(memoryVersion.generation, input.generation), + ), + ); + // 3) bookkeeping — generation column stays the original run id for lookup; + // versions now live under 'live'. Search by generation=runId after + // promote finds nothing (expected); demote restores. + await tx + .update(transformRun) + .set({ + archivedLiveGeneration: archiveGen, + archivedLiveModelKey: priorModelKey, + promotedAt: new Date(), + }) + .where(eq(transformRun.id, run.id)); + }); + } catch (err) { + // Version swap failed after dense activate — restore prior dense target. + if (priorModelKey) { + try { + await activateEmbedModelByKey(client, input.tenantId, priorModelKey); + } catch (restoreErr) { + log.warn( + "promoteGeneration: failed to restore prior embed model after version swap error", + { + priorModelKey, + error: formatCaughtError(restoreErr), + }, + ); + } + } + throw err; + } + return loadTransformRun(deps.db, run.id); } /** * Demote a previously promoted generation: swap archive back to live and - * move the demoted live corpus back onto the run's generation tag. + * move the demoted live corpus back onto the run's generation tag. Restores + * the pre-promote active embed model when one was recorded. */ export async function demoteGeneration( deps: { db: Db; sql: RawSql; config: EngineConfig }, @@ -594,6 +632,20 @@ export async function demoteGeneration( } const archiveGen = run.archivedLiveGeneration; + const priorModelKey = run.archivedLiveModelKey ?? null; + const client = createRawSqlClient(deps.sql); + + // Restore prior dense target first so a missing model_key fails closed + // before we rewrite generation tags. + if (priorModelKey) { + try { + await activateEmbedModelByKey(client, input.tenantId, priorModelKey); + } catch (err) { + throw new TransformPromoteError( + `cannot demote: failed to restore prior embed model ${priorModelKey}: ${formatCaughtError(err)}`, + ); + } + } await deps.db.transaction(async (tx) => { // live (promoted) → back to run generation @@ -620,15 +672,11 @@ export async function demoteGeneration( .update(transformRun) .set({ archivedLiveGeneration: null, + archivedLiveModelKey: null, promotedAt: null, }) .where(eq(transformRun.id, run.id)); }); - // Note: demote does not auto-activate a prior embed model — the host may - // re-activate via a subsequent live capture or explicit promote of another run. - // Live dense search continues against whichever model is currently active; - // vectors for restored live versions remain in their original embed tables. - return loadTransformRun(deps.db, run.id); } diff --git a/src/tools/add.ts b/src/tools/add.ts index 08e4373..552341f 100644 --- a/src/tools/add.ts +++ b/src/tools/add.ts @@ -39,7 +39,7 @@ export const memoryAdd = defineMemoryHttpTool({ id: "@corbits/memory/add", name: "memory_add", description: - "Store a note in tenant memory. Returns { documentId }. " + + "Store a note in tenant memory. Returns { documentId, versionId }. " + "Identity is the authenticated principal on the hub; do not " + "pass tenant or principal ids.", inputSchema: { diff --git a/src/tools/tools.test.ts b/src/tools/tools.test.ts index 09f7fd5..75bc4e1 100644 --- a/src/tools/tools.test.ts +++ b/src/tools/tools.test.ts @@ -115,7 +115,7 @@ describe("createMemoryHttpClient", () => { test("POSTs add under tenant path with Bearer auth", async () => { const { calls, fetchMock } = makeFetchMock(() => ({ status: 200, - json: { documentId: "doc-1" }, + json: { documentId: "doc-1", versionId: "ver-1" }, })); const client = createMemoryHttpClient({ baseUrl: `${BASE}///`, @@ -124,7 +124,7 @@ describe("createMemoryHttpClient", () => { fetch: fetchMock, }); const out = await client.add({ title: "t", text: "body" }); - expect(out).toEqual({ documentId: "doc-1" }); + expect(out).toEqual({ documentId: "doc-1", versionId: "ver-1" }); expect(calls).toHaveLength(1); const c = calls[0]!; expect(c.method).toBe("POST"); @@ -233,7 +233,7 @@ describe("memoryAdd factory", () => { test("happy path: body has no identity fields", async () => { const { calls, fetchMock } = makeFetchMock(() => ({ status: 200, - json: { documentId: "doc-9" }, + json: { documentId: "doc-9", versionId: "ver-9" }, })); const bundle = memoryAdd(toolEnv({ memoryFetch: fetchMock })); expect(bundle.definitions.map((d) => d.name)).toEqual(["memory_add"]); @@ -246,7 +246,7 @@ describe("memoryAdd factory", () => { new AbortController().signal, ); expect(result.isError).toBeFalsy(); - expect(result.content).toBe(JSON.stringify({ documentId: "doc-9" })); + expect(result.content).toBe(JSON.stringify({ documentId: "doc-9", versionId: "ver-9" })); const body = JSON.parse(calls[0]!.body ?? "{}") as Record; expect(body).not.toHaveProperty("tenantId"); expect(body).not.toHaveProperty("principalId"); From bf8d3e14dacf0ac84627700045a4f77faaae34bc Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 10 Aug 2026 09:37:02 -0700 Subject: [PATCH 05/19] Tighten tenant isolation and exclusive embed activation Scope generation search params and appendAccessTags by tenant so cross-tenant ids cannot resolve embed overrides or retag documents. Activate one dense model exclusively; demote with no prior model clears active status instead of leaving the promoted table live. --- IMPLEMENTATION.md | 23 ++++++---- src/core/embed-model-registry.test.ts | 24 +++++++++-- src/core/embed-model-registry.ts | 60 +++++++++++++++++++++------ src/index.ts | 1 + src/memory.ts | 8 ++-- src/ports/fakes.ts | 6 ++- src/ports/types.ts | 2 + src/services/search.ts | 7 +++- src/services/transform.ts | 59 ++++++++++++++++++-------- 9 files changed, 140 insertions(+), 50 deletions(-) diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md index 6e85970..88cbb40 100644 --- a/IMPLEMENTATION.md +++ b/IMPLEMENTATION.md @@ -405,10 +405,12 @@ search); otherwise it throws `MemorySearchInputError` (400). 1. **Generation resolution** — `generation` defaults to `LIVE_GENERATION`. For any non-live generation, `resolveGenerationSearchParams` (transform.ts) - looks up the owning `transform_run` → `transform_config` and pulls its - tuning knobs (`authorityWeight`, `recencyHalfLifeDays`, `mmrLambda`, - `overfetch`, `rerank`); every field it doesn't supply falls back to the - engine's own defaults. Live search never pays for this lookup. + looks up the owning `transform_run` → `transform_config` **for the search + tenant** and pulls its tuning knobs (`authorityWeight`, + `recencyHalfLifeDays`, `mmrLambda`, `overfetch`, `rerank`); every field it + doesn't supply falls back to the engine's own defaults. Live search never + pays for this lookup. Cross-tenant generation ids resolve to `null` + (engine defaults) so embed overrides (including `apiKey`) cannot leak. 2. **Lexical channel** — `fetchLexicalCandidates`: Postgres full-text search (`ts_rank` against `plainto_tsquery` in the configured `FTS_LANGUAGE`, bound as a `regconfig` parameter, over @@ -514,13 +516,16 @@ writes to. A `transform_config` is a named/versioned recipe **Promote / demote (staged cutover):** - `promoteGeneration` — requires `status = 'completed'`. Snapshots the - pre-promote active `model_key`, activates the staged embed model, then - swaps generation tags (`live` → archive, staged → `live`). Records + pre-promote active `model_key`, activates the staged embed model (sole + active for the tenant; peers demoted to `ready`), then swaps generation + tags (`live` → archive, staged → `live`). Records `archived_live_generation` + `archived_live_model_key` for demote. If the - version swap fails after activate, re-activates the prior model_key. + version swap fails after activate, re-activates the prior model_key (or + clears active when there was no prior). - `demoteGeneration` — re-activates `archived_live_model_key` (fail-closed if - the registry row is gone), then restores archive → live and staged corpus - back onto the run generation. + the registry row is gone), or clears all active models when no prior was + recorded, then restores archive → live and staged corpus back onto the run + generation. Plane methods (engine DocumentStore only): `createTransformConfig`, `listTransformConfigs`, `runTransform`, `promoteGeneration`, diff --git a/src/core/embed-model-registry.test.ts b/src/core/embed-model-registry.test.ts index 5869962..de26b05 100644 --- a/src/core/embed-model-registry.test.ts +++ b/src/core/embed-model-registry.test.ts @@ -3,6 +3,7 @@ import type { EmbedClientConfig } from "./embed-client.ts"; import { activateEmbedModel, activateEmbedModelByKey, + clearActiveEmbedModels, computeModelKey, cosineDistanceExpr, DimsOutOfBoundsError, @@ -323,15 +324,29 @@ describe("activateEmbedModel (split)", () => { await ensureEmbedModel(client, "tenant-1", baseConfig, fixtureFetch(768)); const ensureQueries = queries.length; await activateEmbedModel(client, "tenant-1", baseConfig, fixtureFetch(768)); - // activate always issues the status UPDATE after ensure work + // activate always issues the exclusive active UPDATE after ensure work expect( queries.slice(ensureQueries).some((q) => q.sql.includes("SET status = 'active'")), ).toBe(true); + expect( + queries.slice(ensureQueries).some((q) => q.sql.includes("SET status = 'ready'")), + ).toBe(true); + }); +}); + +describe("clearActiveEmbedModels", () => { + it("demotes every active row for the tenant to ready", async () => { + const { client, queries } = createMockClient(); + await clearActiveEmbedModels(client, "tenant-1"); + expect(queries).toHaveLength(1); + expect(queries[0]?.sql).toContain("SET status = 'ready'"); + expect(queries[0]?.sql).toContain("status = 'active'"); + expect(queries[0]?.params).toEqual(["tenant-1"]); }); }); describe("activateEmbedModelByKey", () => { - it("issues UPDATE by model_key without probing the embed endpoint", async () => { + it("selects by model_key then exclusives-activates without probing embed", async () => { const modelKey = "abcdef0123456789"; const queries: Array<{ sql: string; params: readonly unknown[] }> = []; const client: EmbedRegistrySqlClient = { @@ -345,8 +360,9 @@ describe("activateEmbedModelByKey", () => { const result = await activateEmbedModelByKey(client, "tenant-1", modelKey); expect(result.modelKey).toBe(modelKey); expect(result.dims).toBe(768); - expect(queries[0]?.sql).toContain("SET status = 'active'"); - expect(queries[0]?.params).toEqual(["tenant-1", modelKey]); + expect(queries[0]?.sql).toContain("SELECT model_key"); + expect(queries.some((q) => q.sql.includes("SET status = 'ready'"))).toBe(true); + expect(queries.some((q) => q.sql.includes("SET status = 'active'"))).toBe(true); }); it("throws when the registry row is missing", async () => { diff --git a/src/core/embed-model-registry.ts b/src/core/embed-model-registry.ts index 2278a54..47f05b9 100644 --- a/src/core/embed-model-registry.ts +++ b/src/core/embed-model-registry.ts @@ -205,8 +205,10 @@ export async function ensureEmbedModel( /** * Ensure the table exists, then promote this model to the tenant's active - * dense-search target (`status='active'`, `updated_at=now()`). Live capture - * uses this path; transform/replay must use `ensureEmbedModel` only. + * dense-search target (`status='active'`, `updated_at=now()`). Other active + * rows for the tenant are demoted to `ready` so `resolveActiveEmbedTable` + * cannot pick a stale peer by updated_at race. Live capture uses this path; + * transform/replay must use `ensureEmbedModel` only. */ export async function activateEmbedModel( client: EmbedRegistrySqlClient, @@ -215,12 +217,7 @@ export async function activateEmbedModel( fetchImpl: typeof fetch = fetch, ): Promise { const result = await ensureEmbedModel(client, tenantId, config, fetchImpl); - await client.query( - `UPDATE "memory"."embed_model" - SET status = 'active', updated_at = now() - WHERE tenant_id = $1 AND model_key = $2`, - [tenantId, result.modelKey], - ); + await setActiveEmbedModelExclusive(client, tenantId, result.modelKey); return result; } @@ -260,7 +257,8 @@ export async function resolveActiveEmbedTable( * restore the pre-promote live model without needing baseUrl/modelId. * * Throws if the registry row is missing — demote must not silently leave - * dense search on the promoted model. + * dense search on the promoted model. Demotes other active rows for the + * tenant to `ready`. */ export async function activateEmbedModelByKey( client: EmbedRegistrySqlClient, @@ -271,10 +269,8 @@ export async function activateEmbedModelByKey( throw new Error(`activateEmbedModelByKey: invalid modelKey "${modelKey}"`); } const rows = await client.query( - `UPDATE "memory"."embed_model" - SET status = 'active', updated_at = now() - WHERE tenant_id = $1 AND model_key = $2 - RETURNING model_key, model_id, dims`, + `SELECT model_key, model_id, dims FROM "memory"."embed_model" + WHERE tenant_id = $1 AND model_key = $2`, [tenantId, modelKey], ); const row = rows[0]; @@ -283,6 +279,7 @@ export async function activateEmbedModelByKey( `activateEmbedModelByKey: no embed_model row for tenant=${tenantId} model_key=${modelKey}`, ); } + await setActiveEmbedModelExclusive(client, tenantId, modelKey); return { tableName: embeddingTableName(modelKey), dims: row.dims as number, @@ -291,6 +288,43 @@ export async function activateEmbedModelByKey( }; } +/** + * Clear every active embed model for the tenant (status → ready). Used when + * demoting a promote that had no prior live model so dense search degrades + * cleanly instead of remaining on the promoted table after the corpus swap. + */ +export async function clearActiveEmbedModels( + client: EmbedRegistrySqlClient, + tenantId: string, +): Promise { + await client.query( + `UPDATE "memory"."embed_model" + SET status = 'ready', updated_at = now() + WHERE tenant_id = $1 AND status = 'active'`, + [tenantId], + ); +} + +/** Make `modelKey` the sole active model for the tenant. */ +async function setActiveEmbedModelExclusive( + client: EmbedRegistrySqlClient, + tenantId: string, + modelKey: string, +): Promise { + await client.query( + `UPDATE "memory"."embed_model" + SET status = 'ready', updated_at = now() + WHERE tenant_id = $1 AND status = 'active' AND model_key <> $2`, + [tenantId, modelKey], + ); + await client.query( + `UPDATE "memory"."embed_model" + SET status = 'active', updated_at = now() + WHERE tenant_id = $1 AND model_key = $2`, + [tenantId, modelKey], + ); +} + /** * Resolve any registered embed table by model_key (active or ready). * Used when searching a non-live generation whose transform_config embeds diff --git a/src/index.ts b/src/index.ts index 596104c..e0991fa 100644 --- a/src/index.ts +++ b/src/index.ts @@ -115,6 +115,7 @@ export { export { activateEmbedModel, activateEmbedModelByKey, + clearActiveEmbedModels, ensureEmbedModel, resolveActiveEmbedTable, resolveEmbedTableByModelKey, diff --git a/src/memory.ts b/src/memory.ts index a24b004..64b23db 100644 --- a/src/memory.ts +++ b/src/memory.ts @@ -705,7 +705,7 @@ function createPlaneFromStore( if (peers && peers.length > 0) { const docTag = documentTag(result.documentId); if (store.appendAccessTags) { - await store.appendAccessTags(result.documentId, [docTag]); + await store.appendAccessTags(params.tenantId, result.documentId, [docTag]); } else { log.warn( "memory.add: share.principals set but DocumentStore has no appendAccessTags; peer grants may not match", @@ -993,9 +993,10 @@ function createEngineDocumentStore(config: MemoryConfig): { }; }, - async appendAccessTags(documentId, tags) { + async appendAccessTags(tenantId, documentId, tags) { if (tags.length === 0) return; - // Union into existing access_tags array (postgres text[]). + // Union into existing access_tags array (postgres text[]). Tenant + // filter is defense-in-depth against a forged documentId. await sql` UPDATE "memory"."document" SET access_tags = ( @@ -1007,6 +1008,7 @@ function createEngineDocumentStore(config: MemoryConfig): { ) ) WHERE id = ${documentId} + AND tenant_id = ${tenantId} `; }, diff --git a/src/ports/fakes.ts b/src/ports/fakes.ts index e1ecf0e..bfe31c1 100644 --- a/src/ports/fakes.ts +++ b/src/ports/fakes.ts @@ -89,8 +89,10 @@ export function createFakeDocumentStore(): DocumentStore { return { documentId, versionId: `fake_ver_${seq}` }; }, - async appendAccessTags(documentId, tags) { - const row = docs.find((d) => d.documentId === documentId); + async appendAccessTags(tenantId, documentId, tags) { + const row = docs.find( + (d) => d.documentId === documentId && d.tenantId === tenantId, + ); if (!row) return; const set = new Set(row.accessTags); for (const t of tags) set.add(t); diff --git a/src/ports/types.ts b/src/ports/types.ts index 651003e..8ef1d08 100644 --- a/src/ports/types.ts +++ b/src/ports/types.ts @@ -111,8 +111,10 @@ export type DocumentStore = { * Append access tags after insert (used by share materialization to stamp * `memory.doc:` once the document id is known). Optional — stores that * omit it leave peer share grants without a matching tag (fail-closed). + * Tenant is required so a forged documentId cannot retag another tenant. */ appendAccessTags?( + tenantId: string, documentId: string, tags: readonly string[], ): Promise; diff --git a/src/services/search.ts b/src/services/search.ts index d78978c..67f3037 100644 --- a/src/services/search.ts +++ b/src/services/search.ts @@ -784,7 +784,12 @@ export async function hybridSearch( const resolvedTuning = generation === LIVE_GENERATION ? null - : await resolveGenerationSearchParams(db, generation, config.embed); + : await resolveGenerationSearchParams( + db, + generation, + config.embed, + tenantId, + ); const authorityWeight = resolvedTuning?.authorityWeight ?? AUTHORITY_WEIGHT; const recencyHalfLifeMs = diff --git a/src/services/transform.ts b/src/services/transform.ts index cba261c..5e33adf 100644 --- a/src/services/transform.ts +++ b/src/services/transform.ts @@ -23,7 +23,7 @@ import { EmbedClientConfigSchema, type EmbedClientConfig } from "../core/embed-c import type { RerankClientConfig } from "../core/rerank-client.ts"; import { deriveFromRawCapture, type CaptureInput } from "./capture.ts"; import { LIVE_GENERATION } from "../core/generation.ts"; -import { activateEmbedModel, activateEmbedModelByKey, resolveActiveEmbedTable } from "../core/embed-model-registry.ts"; +import { activateEmbedModel, activateEmbedModelByKey, clearActiveEmbedModels, resolveActiveEmbedTable } from "../core/embed-model-registry.ts"; import { createRawSqlClient } from "../core/embed-sql.ts"; export class TransformConfigNotFoundError extends Error { @@ -245,21 +245,29 @@ export interface GenerationSearchParams { // Resolves a search-time `generation` (a transform_run id, per the 1:1 // `transform_run.generation` uniqueness) back to its config's tuning knobs. -// Returns `null` when the generation isn't a known replay run (including -// 'live', which the caller should never even ask this for) — hybridSearch -// falls back to its own engine defaults for every field in that case. +// Returns `null` when the generation isn't a known replay run for this tenant +// (including 'live', which the caller should never even ask this for) — +// hybridSearch falls back to its own engine defaults for every field in that +// case. Tenant is required so a cross-tenant generation id cannot resolve +// another tenant's embed overrides (which may carry apiKey). // // `engineEmbed` is required to fully resolve a partial transform embed // override (same merge rules as runTransform). export async function resolveGenerationSearchParams( db: Db, generation: string, - engineEmbed?: EngineConfig["embed"], + engineEmbed: EngineConfig["embed"] | undefined, + tenantId: string, ): Promise { const runRows = await db .select({ configId: transformRun.configId }) .from(transformRun) - .where(eq(transformRun.generation, generation)) + .where( + and( + eq(transformRun.generation, generation), + eq(transformRun.tenantId, tenantId), + ), + ) .limit(1); const run = runRows[0]; if (!run) return null; @@ -267,7 +275,12 @@ export async function resolveGenerationSearchParams( const configRows = await db .select({ params: transformConfig.params }) .from(transformConfig) - .where(eq(transformConfig.id, run.configId)) + .where( + and( + eq(transformConfig.id, run.configId), + eq(transformConfig.tenantId, tenantId), + ), + ) .limit(1); const configRow = configRows[0]; if (!configRow) return null; @@ -534,6 +547,11 @@ export async function promoteGeneration( } const configRow = await loadTransformConfig(deps.db, run.configId); + if (configRow.tenantId !== input.tenantId) { + throw new TransformPromoteError( + `transform_config tenant mismatch for generation ${input.generation}`, + ); + } const embed = buildEmbedClientConfig(configRow.params.embed, deps.config.embed); const archiveGen = `archive_${run.id}_${Date.now()}`; const client = createRawSqlClient(deps.sql); @@ -581,18 +599,20 @@ export async function promoteGeneration( }); } catch (err) { // Version swap failed after dense activate — restore prior dense target. - if (priorModelKey) { - try { + try { + if (priorModelKey) { await activateEmbedModelByKey(client, input.tenantId, priorModelKey); - } catch (restoreErr) { - log.warn( - "promoteGeneration: failed to restore prior embed model after version swap error", - { - priorModelKey, - error: formatCaughtError(restoreErr), - }, - ); + } else { + await clearActiveEmbedModels(client, input.tenantId); } + } catch (restoreErr) { + log.warn( + "promoteGeneration: failed to restore prior embed model after version swap error", + { + priorModelKey, + error: formatCaughtError(restoreErr), + }, + ); } throw err; } @@ -636,7 +656,8 @@ export async function demoteGeneration( const client = createRawSqlClient(deps.sql); // Restore prior dense target first so a missing model_key fails closed - // before we rewrite generation tags. + // before we rewrite generation tags. No prior model → clear active so + // dense degrades rather than remaining on the promoted table after swap. if (priorModelKey) { try { await activateEmbedModelByKey(client, input.tenantId, priorModelKey); @@ -645,6 +666,8 @@ export async function demoteGeneration( `cannot demote: failed to restore prior embed model ${priorModelKey}: ${formatCaughtError(err)}`, ); } + } else { + await clearActiveEmbedModels(client, input.tenantId); } await deps.db.transaction(async (tx) => { From 1fc230b2e858d91c9ae682e5c8b710c94ea1380d Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 10 Aug 2026 10:03:19 -0700 Subject: [PATCH 06/19] =?UTF-8?q?Use=20DATABASE=5FURL=20and=20rename=20Pos?= =?UTF-8?q?tgres=20schema=20knowledge=20=E2=86=92=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hosts pass memory.databaseUrl or DATABASE_URL (same Postgres as the hub is fine); tables live under the memory schema. KNOWLEDGE_DATABASE_URL remains a deprecated alias. Fresh installs only for the schema rename. --- ARCHITECTURE.md | 3 ++- CHANGELOG.md | 9 +++++++-- src/mount-config.test.ts | 4 +++- src/mount-config.ts | 6 +++++- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index d56cbac..0ddd336 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -9,7 +9,8 @@ plane and the protected routes that read and write it. The store was detachable from a larger backend, then mountable: - No memory table has a foreign key into any control-plane table — cross-refs - (`tenant_id`, `principal_id`, source refs) are plain `text`. + (`tenant_id`, `principal_id`, source refs) are plain `text`. Tables live in + the **`memory`** schema (same Postgres URL as the host is fine). - Embedding and reranking go out as plain HTTP to configured model endpoints. - Document access is Interchange grant tags on the row (`accessTags` + creator), not a private ACL engine inside this package. diff --git a/CHANGELOG.md b/CHANGELOG.md index 8006dbe..f4b5a7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,14 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -### Added +### Changed +- **Breaking:** Postgres schema renamed from `knowledge` to **`memory`**. Fresh + installs only — drop/recreate the old schema (or rename) on existing DBs. + Citation `open.type` is now `"memory"`. +- **Breaking:** env var is `DATABASE_URL` (was `KNOWLEDGE_DATABASE_URL`); pass + `memory.databaseUrl` on config as an alternative. No deprecated alias. - Interchange `defineTool` factories at `@corbits/memory/tools` (`memory_add`, `memory_search`, `memory_list`) — HTTP clients for mounted hub routes with install env `memoryBaseUrl` / `memoryTenantId` / `memoryAuthToken`. Declared via `package.json` `interchange.tools` and `exports["./tools"]`. -### Changed +### Previously - **Breaking:** package and public surface renamed from `@corbits/knowledge-engine` to `@corbits/memory`. Public APIs: `createMemory` (optional `app` registers HTTP), diff --git a/src/mount-config.test.ts b/src/mount-config.test.ts index 5335262..28971ca 100644 --- a/src/mount-config.test.ts +++ b/src/mount-config.test.ts @@ -54,6 +54,8 @@ describe("loadMemoryConfig — EMBED_TIMEOUT_MS / RERANK_TIMEOUT_MS", () => { it("rejects a non-positive-integer EMBED_TIMEOUT_MS", () => { process.env.EMBED_TIMEOUT_MS = "not-a-number"; - expect(() => loadMemoryConfig()).toThrow("EMBED_TIMEOUT_MS must be a positive integer"); + expect(() => loadMemoryConfig()).toThrow( + "EMBED_TIMEOUT_MS must be a positive integer", + ); }); }); diff --git a/src/mount-config.ts b/src/mount-config.ts index 350929d..df608f4 100644 --- a/src/mount-config.ts +++ b/src/mount-config.ts @@ -49,7 +49,11 @@ function optionalIntEnv(name: string): number | undefined { /** * Build a config from environment variables — a convenience for env-driven - * deploys. Hosts may also construct `MemoryConfig` programmatically. + * deploys. Hosts may also construct `MemoryConfig` programmatically (pass + * `memory.databaseUrl` directly — no env required). + * + * Database URL resolution for env-driven loads: DATABASE_URL — same Postgres + * as the host is fine; tables live under the `memory` schema, not public. */ export function loadMemoryConfig(): MemoryConfig { return { From d37de426bb3d8dddc1b3b2455f57b70fd2b040aa Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 10 Aug 2026 10:15:28 -0700 Subject: [PATCH 07/19] Fix dense entity filter after memory schema rename Dense search with entityIds still queried knowledge_edge after the schema moved to memory.edge. Point the subquery at memory.edge, assert the table name in tests, drop legacy knowledge_embed_model mock matches, and align CHANGELOG/IMPLEMENTATION with versionId on add and host- privileged transform APIs. Round-2 multi-lens review + convergence notes under .corbits/. --- .corbits/review-round2-bruckheimer.md | 41 +++++++++++++ .corbits/review-round2-convergence.md | 88 +++++++++++++++++++++++++++ .corbits/review-round2-critique.md | 74 ++++++++++++++++++++++ .corbits/review-round2-gaasbot.md | 44 ++++++++++++++ .corbits/review-round2-greybeard.md | 59 ++++++++++++++++++ .corbits/review-round2-neckbeard.md | 32 ++++++++++ .corbits/review-round2-oss.md | 53 ++++++++++++++++ .corbits/review-round2-schema.md | 60 ++++++++++++++++++ .corbits/review-round2-security.md | 72 ++++++++++++++++++++++ CHANGELOG.md | 7 ++- IMPLEMENTATION.md | 10 ++- src/services/search.test.ts | 50 ++++++++++----- src/services/search.ts | 2 +- 13 files changed, 573 insertions(+), 19 deletions(-) create mode 100644 .corbits/review-round2-bruckheimer.md create mode 100644 .corbits/review-round2-convergence.md create mode 100644 .corbits/review-round2-critique.md create mode 100644 .corbits/review-round2-gaasbot.md create mode 100644 .corbits/review-round2-greybeard.md create mode 100644 .corbits/review-round2-neckbeard.md create mode 100644 .corbits/review-round2-oss.md create mode 100644 .corbits/review-round2-schema.md create mode 100644 .corbits/review-round2-security.md diff --git a/.corbits/review-round2-bruckheimer.md b/.corbits/review-round2-bruckheimer.md new file mode 100644 index 0000000..1e7e6e0 --- /dev/null +++ b/.corbits/review-round2-bruckheimer.md @@ -0,0 +1,41 @@ +# Bruckheimer — PR #31 round 2 (HEAD `6189b56`) + +## One-liner + +Foundation so a company-brain distiller can write **inferred claims** with +provenance, rank them in time, re-distill offline without trashing live search, +and share docs with real grants — not just tags. + +## Hook progress + +| Piece | Status for the hook | +|-------|---------------------| +| Claim-bearing / derived_from / provenance | Shipped | +| Temporal classes + validity | Shipped | +| Staged transform / promote / demote | Shipped (code); weak tests | +| Share grants materialization | Shipped (fail-soft without writable store) | +| Capture feed / distiller workflow | **Not this PR** (CL-5868/5869) | +| Schema/config host-friendly | DATABASE_URL + memory schema — good | + +## Product blockers + +1. **Dense entity filter broken after rename** — if any host/search UI filters by + entity, dense path dies. Fix before merge. +2. **Promote rollback untested** — if demote is the safety valve for bad + distillation, untested demote is a product risk, not just eng debt. +3. **Share without WritableGrantStore** only warns — hosts will think share + “worked.” Consider fail-loud option or return receipt with + `grantsMaterialized: false` (follow-up OK). + +## Breaking-change cost/benefit + +- `knowledge` → `memory` schema: right name for the product; cost is fresh + install only. Acceptable if no production tenants on old schema. +- `DATABASE_URL` preferred: lowers host friction (one DB). Risk: host with two + URLs silently picks the wrong one — document clearly (done in mount-config). + +## Ship advice + +Fix the dense SQL bug, clarify CHANGELOG on versionId, merge as foundation. +Do not hold the PR for the distiller itself. Next product milestone is feed + +workflow, not more schema polish. diff --git a/.corbits/review-round2-convergence.md b/.corbits/review-round2-convergence.md new file mode 100644 index 0000000..dc84bb3 --- /dev/null +++ b/.corbits/review-round2-convergence.md @@ -0,0 +1,88 @@ +# Convergence review — PR #31 round 2 (HEAD `6189b56`) + +**Method:** 8 deep lenses in-session (fleet spawn blocked: Codex profile +`fleur` unauthorized). Lenses: critique, greybeard, gaasbot, neckbeard, +bruckheimer, security, schema, OSS. Artifacts: +`.corbits/review-round2-*.md`. + +## Converged verdict + +**CHANGES REQUESTED — one hard blocker, then merge-eligible.** + +All eight lenses agree the distillation foundation is the right shape and that +prior promote/demote/versionId work improved the bar. All eight that touched +search/schema flag the same ship-blocker. + +--- + +## Must-fix before merge (unanimous / multi-lens) + +| ID | Finding | Lenses | Action | +|----|---------|--------|--------| +| **M1** | Dense entity filter SQL still uses `knowledge_edge`; table is `"memory"."edge"` (`src/services/search.ts:537`) | critique, greybeard, gaasbot, neckbeard, security, schema, OSS, bruckheimer | Fix SQL + add regression test (dense path with entityIds) | +| **M2** | CHANGELOG Unreleased “Previously” claims add returns only `{ documentId }`; code returns `versionId` | critique, gaasbot, greybeard, OSS | Correct CHANGELOG | + +## Should-fix (same PR if cheap; else ticket) + +| ID | Finding | Lenses | Action | +|----|---------|--------|--------| +| **S1** | No promote/demote service E2E regression | critique, greybeard, gaasbot, bruckheimer | Add transform test: ensure→promote→demote restores model_key + generation | +| **S2** | search.test mocks still accept `FROM knowledge_embed_model` | neckbeard, OSS | Match only `"memory"."embed_model"` | +| **S3** | Document that transform/promote/demote are host-privileged (no principal on API, no HTTP) | security, gaasbot, greybeard | Short IMPLEMENTATION / AUTHZ note | + +## Follow-up (do not block merge) + +| ID | Finding | Lenses | +|----|---------|--------| +| F1 | `setActiveEmbedModelExclusive` two-step race | critique, security | +| F2 | Promote/demote multi-step windows; consider tenant advisory lock | critique, greybeard, security | +| F3 | Share fail-soft without WritableGrantStore / appendAccessTags — receipt field | critique, bruckheimer, security | +| F4 | TS `knowledge*` identifiers + migration filenames + comment rot | neckbeard, schema | +| F5 | Ops note for `ALTER SCHEMA knowledge RENAME TO memory` if any old install | greybeard, schema | +| F6 | README advanced surface (transform exports) | OSS, bruckheimer | + +## Explicit non-goals this PR + +- Distiller workflow / capture feed (CL-5868/5869) +- HTTP routes for transform +- Bulk rename of knowledge* TypeScript symbols +- Re-introducing separate knowledge DB + +## Steelman of “merge as-is” + +Tests are green; entityIds on dense may be rare; rename is fresh-install-only. +**Rejected:** a single untested raw-SQL island after a schema rename is exactly +the class of bug that survives CI and fails first real use. Fix is trivial. + +## Steelman of “hold for promote E2E” + +Demote is the safety valve for bad distillation. **Partial accept:** S1 is +high value but not a correctness hole in the current code path (demote restore +exists). Prefer same-PR if <1h; else ticket linked from PR. + +## Converged fix order + +1. **M1** fix + test +2. **M2** CHANGELOG +3. **S2** tighten mocks (with M1 test) +4. **S1** if time +5. **S3** one paragraph +6. Re-run typecheck + test → push + +## Bar after fixes + +| Bar | Status after M1+M2 | +|-----|--------------------| +| Security | Pass (with S3 doc preferred) | +| Product | Pass foundation | +| Greybeard / architecture | Pass | +| OSS quality | Pass pre-1.0 | +| Critique | Pass with S1 follow-up | + +--- + +## Note on fleet + +All 8 `task` spawns failed: `Codex profile "fleur" is not authorized`. Reviews +were executed in-session with the same multi-lens briefs. Re-auth `/model` +(profile fleur) to restore sub-agent fleet for future rounds. diff --git a/.corbits/review-round2-critique.md b/.corbits/review-round2-critique.md new file mode 100644 index 0000000..7e4ae77 --- /dev/null +++ b/.corbits/review-round2-critique.md @@ -0,0 +1,74 @@ +# Critique — PR #31 round 2 (HEAD `6189b56`) + +In-session (fleet blocked: Codex profile `fleur`). Diff `origin/main..HEAD`. +Typecheck clean; 367 tests green at last gate. + +## Verdict + +**CHANGES REQUESTED.** Prior promote/demote and versionId fixes landed, but the +`knowledge` → `memory` rename left a **live dense-path SQL bug**, and +promote/demote still lack end-to-end regression tests. + +## Critical + +1. **Dense entity filter still queries `knowledge_edge`** + (`src/services/search.ts:537`). After schema rename, the table is + `"memory"."edge"`. Any dense search with `entityIds` will fail at runtime + (`relation "knowledge_edge" does not exist`). Lexical path is fine (Drizzle + `knowledgeEdge` → `memory.edge`). No test exercises dense+entityIds. + +## High + +2. **No automated promote → demote → dense restore E2E.** Registry unit tests + cover ensure/activate/byKey, but nothing asserts: staged run under model B + does not flip live active; promote swaps generation + activates staged; + demote restores generation **and** prior model_key. CL-5872 rollback + acceptance still untested at the service layer. + +3. **Promote/demote are multi-step outside a single transaction.** Activate + dense, then version swap (or reverse on demote). Concurrent promote of two + generations, or crash mid-window, can leave dense target and generation tags + briefly inconsistent. Documented preference is intentional; still a + production footgun without locks / single-flight. + +## Medium + +4. **`setActiveEmbedModelExclusive` is two non-atomic UPDATEs** + (`embed-model-registry.ts:309-325`). Concurrent activate of A and B can + leave two `active` rows until next exclusive call; `ORDER BY updated_at` + picks one, but window exists. + +5. **Transform plane methods take only `tenantId` / `configId` — no principal.** + In-process API; no HTTP routes. Correct for library shape, but any host that + re-exports without its own grant check hands promote/demote to any caller + who can reach the plane. Docs should state “host must authorize.” + +6. **Share path still fail-soft** when `appendAccessTags` or WritableGrantStore + missing (`memory.ts:707-727`): warns, continues, peers fail-closed. Easy to + miss in production. + +7. **CHANGELOG drift:** Unreleased “Previously” still says `add` returns + `{ documentId }` only; wire now returns `{ documentId, versionId }`. + +## Low / Nits + +8. Comments and enums still say `knowledge.version` / `knowledge.embed_model` + (`enums.ts`, `embed-model-registry.ts:34`, `generation.ts`). +9. Migration file still named `0002_knowledge_baseline.sql` while creating + `memory.*`. +10. TS exports still `knowledgeDocument` / `knowledgeVersion` under memory schema. + +## Test gaps + +- Dense search + `entityIds` (would catch Critical #1). +- `promoteGeneration` / `demoteGeneration` service tests with fake SQL + version rows. +- Concurrent exclusive activate (optional stress). +- `loadMemoryConfig` DATABASE_URL vs KNOWLEDGE preference already covered. + +## Assumptions challenged + +- “Rename was complete because migrations and drizzle use memory” — raw SQL + island in dense path was not. +- “367 green ⇒ rename safe” — unit tests mock embed_model with dual match + (`knowledge_embed_model` OR `memory.embed_model`) and never hit entity filter + raw SQL. diff --git a/.corbits/review-round2-gaasbot.md b/.corbits/review-round2-gaasbot.md new file mode 100644 index 0000000..c1d1735 --- /dev/null +++ b/.corbits/review-round2-gaasbot.md @@ -0,0 +1,44 @@ +# Gaasbot (CTO) — PR #31 round 2 (HEAD `6189b56`) + +## CTO verdict + +**Right foundation, one ship-blocker.** This is the correct shape for +resident distillation: claim-bearing + temporal + staged transform + grants. +Do not expand scope into the distiller workflow (CL-5869) on this PR. + +## Must-fix-before-merge + +1. Fix dense-path `knowledge_edge` → `"memory"."edge"` (`search.ts:537`). +2. Add a regression test that would fail on that bug (dense fetch with + entityIds, assert SQL contains `"memory"."edge"` or run against real SQL + mock that only knows memory.edge). +3. Fix CHANGELOG: add returns `versionId`; remove contradictory “Previously” + line or mark superseded. + +## Can-ship-with-followups + +- Promote/demote service-level tests. +- Tenant-scoped advisory lock on promote/demote. +- Atomic exclusive activate (single SQL CTE or transaction). +- Explicit “host authorizes transform APIs” note in IMPLEMENTATION.md. +- Rename TS `knowledge*` symbols in a dedicated PR (not this one). + +## Defer + +- HTTP routes for transform/promote (in-process is fine for v1 distiller). +- Capture feed (CL-5868), relevancy (CL-5867), retention (CL-5871). +- Upgrade migration from `knowledge` schema for old DBs unless a customer exists. + +## Architecture notes + +- Exporting transform + embed registry from package root is aggressive but OK + for the distiller as first-party consumer. Keep them off HTTP until grants + exist. +- Preferring `DATABASE_URL` is correct for “same Postgres, own schema.” Warn + hosts that still set both URLs with different values — preferred wins. +- Do not re-introduce a separate knowledge DB requirement. + +## Priority + +Blocker fix is a one-liner + test. Merge after that; iterate on promote +hardening in the same branch if cheap, else follow-up ticket. diff --git a/.corbits/review-round2-greybeard.md b/.corbits/review-round2-greybeard.md new file mode 100644 index 0000000..7af7025 --- /dev/null +++ b/.corbits/review-round2-greybeard.md @@ -0,0 +1,59 @@ +# Greybeard — PR #31 round 2 (HEAD `6189b56`) + +## Verdict + +**HOLD for one correctness fix; then ship foundation.** Architecture of +ensure-vs-activate, claim-bearing, temporal classes, and share materialization +is sound. Schema rename + DATABASE_URL is the right long-term shape. + +## Ship / hold + +**Hold** until dense `entityIds` SQL is fixed (`knowledge_edge` → `"memory"."edge"`). +After that: **ship with follow-ups** (promote E2E tests, exclusive activate +transaction, host authz docs for transform). + +## Critical / High + +1. **Raw SQL residue after schema rename** — `search.ts:537` `knowledge_edge`. + Irreversible-looking renames that leave one path broken are worse than no + rename: green CI + red prod. + +2. **Fresh-install-only schema rename** is honest in CHANGELOG but operationally + harsh. Acceptable for pre-1.0 / no prod tenants; document a one-shot + `ALTER SCHEMA knowledge RENAME TO memory` + table renames for anyone who + already migrated under `knowledge`. + +## Medium — design debt (acceptable for now) + +3. **JS identifiers lag schema** (`knowledgeDocument` table → `document`). Fine + if intentional transitional; pick a rename PR later — do not half-rename. +4. **Embedding tables keyed only by model_key**, multi-tenant rows inside. + Tenant filter on every dense query is load-bearing; keep that invariant in + review checklist forever. +5. **Promote activate-then-swap** preference is documented and reasonable. + Prefer advisory lock per tenant around promote/demote before multi-tenant + production load. +6. **Docs generally lockstep** with DATABASE_URL / memory schema after last + commit; IMPLEMENTATION table and AGENTS.md match. CHANGELOG “Previously” + still contradicts versionId on add. + +## Doc drift + +| Claim | Reality | +|-------|---------| +| CHANGELOG: add returns `{ documentId }` | Returns `{ documentId, versionId }` | +| Comments: knowledge.embed_model | Table is memory.embed_model | +| open.type "memory" | Correct in search.ts | + +## Design decisions that aged well + +- `ensureEmbedModel` vs `activateEmbedModel` split (replay must not steal live). +- `activateEmbedModelByKey` for demote without re-probe. +- `archived_live_model_key` on transform_run. +- Share grants + pass-through condition registry. +- Enum lockstep test (enums.lockstep.test.ts). + +## Recommendation + +Fix Critical SQL → add dense+entityIds test → optional promote E2E → merge. +Do not block on knowledge* TypeScript renames. diff --git a/.corbits/review-round2-neckbeard.md b/.corbits/review-round2-neckbeard.md new file mode 100644 index 0000000..ff03a4b --- /dev/null +++ b/.corbits/review-round2-neckbeard.md @@ -0,0 +1,32 @@ +# Neckbeard — PR #31 round 2 (HEAD `6189b56`) + +## Real bugs hiding as nits + +1. **`knowledge_edge` in raw SQL** (`search.ts:537`) — not a naming nit. **Bug.** +2. **search.test.ts still accepts `FROM knowledge_embed_model`** as a success + path for mocks (`search.test.ts:237, 412`). Teaches the wrong table name; + should only match `"memory"."embed_model"`. + +## Naming debt inventory (cosmetic unless noted) + +| Residue | Severity | +|---------|----------| +| `knowledgeDocument`, `knowledgeVersion`, `knowledgeChunk`, `knowledgeEdge`, `knowledgeEntity`, `knowledgeEmbedModel` exports | Cosmetic / API-internal | +| `KNOWLEDGE_SCHEMA` deprecated alias | OK transitional | +| `migrations/0002_knowledge_baseline.sql` filename | Cosmetic; content correct | +| Comments `knowledge.version`, `knowledge.embed_model` | Doc rot | +| Id prefixes `kver`, `kdoc` | Cosmetic | +| grant-tags test still uses `knowledge.project:ke` as a free-form tag | Fine (host tags) | +| CHANGELOG “Postgres schema name remains knowledge” removed; good | — | + +## Nits + +- Dual match in tests for old embed_model table should die with the rename. +- `### Previously` in CHANGELOG is nonstandard Keep-a-Changelog structure. +- Package still says “knowledge plane” in a few comments (`config.ts` FTS). +- `openTarget` comment still says “generic knowledge doc” (`search.ts:156`). + +## What not to rewrite + +Do not rename all `knowledge*` TS symbols in this PR. Ship the SQL fix and +stop. A bulk rename PR with codemod is fine later. diff --git a/.corbits/review-round2-oss.md b/.corbits/review-round2-oss.md new file mode 100644 index 0000000..890f2a1 --- /dev/null +++ b/.corbits/review-round2-oss.md @@ -0,0 +1,53 @@ +# OSS / public API quality — PR #31 round 2 (HEAD `6189b56`) + +## Public surface inventory (package root) + +- `createMemory`, `loadMemoryConfig`, `runMemoryMigrations`, `MemoryError` +- Routes: `registerMemoryRoutes` +- Ports: DocumentStore types, fakes, WritableGrantStore +- Share: buildShareGrants, materializeShareGrants, MEMORY_SHARE_* +- Transform: createTransformConfig, runTransform, promote/demoteGeneration, … +- Embed registry: ensure/activate/resolve helpers +- Degrade metrics, FTS helpers + +**Note:** Transform + embed registry on the root export is a large surface for +an “add/search/list” product blurb. Acceptable for distiller-as-consumer; +README should mention advanced APIs. + +## Breaking changes completeness + +| Change | CHANGELOG | Code | +|--------|-----------|------| +| Schema knowledge → memory | Yes | Yes | +| DATABASE_URL preferred | Yes | Yes | +| open.type memory | Yes | Yes | +| add returns versionId | **Stale “Previously” says no** | Yes | +| Claim-bearing / temporal / transform | Partial (IMPLEMENTATION) | Yes | + +## Quality bar + +| Area | Pass? | Notes | +|------|-------|-------| +| arktype at edges | Pass | transform params, raw_capture replay | +| Enum lockstep tests | Pass | enums.lockstep.test.ts | +| Tenant SQL discipline | Pass* | *except broken edge table name | +| Module focus | Pass | services split reasonably | +| Docs match exports | Partial | CHANGELOG versionId; comments knowledge.* | +| Test coverage public contracts | Partial | no promote E2E; dense+entityIds missing | +| Semver honesty | Partial | fix Unreleased Previously section | + +## Must-fix for OSS merge + +1. Dense entity SQL table name. +2. CHANGELOG honesty on `MemoryAddResult` / wire body. +3. Regression test for (1). + +## Nice-to-have before wider publish + +- README section: transform/promote privileged, host-gated. +- Drop dual mock match for `knowledge_embed_model` in tests. +- Do not bulk-rename knowledge* TS identifiers in this PR. + +## Verdict + +**Fail OSS bar until Critical SQL + CHANGELOG; then pass for pre-1.0 foundation.** diff --git a/.corbits/review-round2-schema.md b/.corbits/review-round2-schema.md new file mode 100644 index 0000000..d8bb692 --- /dev/null +++ b/.corbits/review-round2-schema.md @@ -0,0 +1,60 @@ +# Schema / migrations — PR #31 round 2 (HEAD `6189b56`) + +## Summary + +Migrations and Drizzle schema consistently use `"memory".…` with short table +names (`document`, `version`, `chunk`, `edge`, …). Config prefers DATABASE_URL. +One application raw-SQL path still names the pre-rename edge table. + +## Critical + +1. **App SQL vs migration mismatch:** `search.ts:537` uses `knowledge_edge`; + migrations create `"memory"."edge"`. Dense entity filter is broken on any + real Postgres. + +## High + +2. **No upgrade path** from prior `knowledge` schema installs — CHANGELOG says + fresh-only. Correct if intentional; add a short ops note (RENAME SCHEMA + + RENAME tables) if anyone already applied old branch migrations. + +3. **Migration filenames** still `0002_knowledge_baseline.sql` etc. Content is + memory.*; confusing for operators grepping filenames. Optional rename of + files is risky if migration ledger already records names — leave filenames, + fix comments at top of 0002. + +## Medium + +4. **`0001_extensions.sql`** must create schema `memory` before 0002 — verify + CREATE SCHEMA IF NOT EXISTS memory (assumed present; was part of rename). +5. **`archived_live_model_key`** text, no FK to embed_model — intentional + (model row may be demoted/deleted); demote fails closed if key missing. +6. **embed_model ON CONFLICT** updates dims/model_id without status change — + correct for ensure. +7. **CHECK/arktype lockstep** covered by `enums.lockstep.test.ts` — good. +8. **Internal FKs** document ← version ← chunk; edge/entity free of control- + plane FKs — matches AGENTS.md. +9. **DATABASE_URL resolution** order correct; tests cover prefer / fallback / + throw. + +## Low + +10. Index names dropped `knowledge_` prefix — good. +11. Dynamic embedding tables: FK to memory.chunk; tenant_id column; no per- + tenant table isolation (by design). + +## Config + +| Source | Behavior | +|--------|----------| +| `memory.databaseUrl` on config | Programmatic hosts | +| `DATABASE_URL` | Preferred env | +| `KNOWLEDGE_DATABASE_URL` | Deprecated alias | +| Neither | throw | + +## Ranked defects + +1. Critical: raw SQL `knowledge_edge` +2. High: document upgrade path or confirm zero external installs +3. Medium: 0002 file header still says “knowledge plane” +4. Low: TS knowledge* symbols / comment rot diff --git a/.corbits/review-round2-security.md b/.corbits/review-round2-security.md new file mode 100644 index 0000000..7c9124c --- /dev/null +++ b/.corbits/review-round2-security.md @@ -0,0 +1,72 @@ +# Security — PR #31 round 2 (HEAD `6189b56`) + +## Summary + +Tenant isolation on SQL paths is generally solid (tenant_id first; grant post- +filter). No auth in package (by design). One correctness issue can cause +hard-fail (DoS of entity-filtered dense search). Transform APIs are privileged +operations without built-in principal checks. + +## Critical + +None for classic IDOR/authz bypass found in this pass. Closest: + +**C-adjacent:** Dense `entityIds` query references non-existent `knowledge_edge` +(`search.ts:537`) — availability/DoS of that code path, not data leak. + +## High + +1. **Transform promote/demote/run lack principal binding** + (`memory.ts:751-798`, `transform.ts`). Anyone who can call the in-process + plane with a tenantId can rewrite that tenant’s live generation and active + embed model. Mitigation: host-only, no HTTP. **Requirement:** document that + hosts must gate these like admin APIs; never expose unauthenticated. + +2. **Shared embedding physical tables across tenants** (table name = + `embedding_` only). Isolation is `WHERE tenant_id = $1` on every + dense query. A missing tenant predicate on a future query is cross-tenant + vector leak. Current dense SQL includes `e.tenant_id = $1 AND c.tenant_id = $1`. + **Keep as permanent review invariant.** + +## Medium + +3. **Exclusive activate race** — two concurrent activates can briefly leave two + active rows; resolve picks latest updated_at. Unlikely privilege issue; + wrong model for search is integrity issue. + +4. **Share grants use `origin: "system"`** (`share-grants.ts:92`) with + always-true condition evaluator. Correct for not fail-closing, but grants + look “system-minted.” Audit trail relies on `conditions.memoryShare` payload. + Ensure host UIs show that payload. + +5. **Promote activate-before-swap window** — dense points at new model while + live versions still old (or reverse on demote). Transient wrong hits, not + cross-tenant. + +6. **`appendAccessTags` optional** — if missing, peer grants may not match tags; + peers fail-closed (safe) but sharer believes share succeeded. + +## Low + +7. Model endpoint URLs trusted by design (AGENTS.md) — no SSRF filter. Host + responsibility. +8. Dynamic SQL for embed table names validated by `EMBED_TABLE_NAME_PATTERN` / + modelKey hex — good. +9. Dynamic `dims` interpolated only after integer bounds check — good. + +## Recommended fixes + +| # | Fix | +|---|-----| +| 1 | `"memory"."edge"` in dense entity SQL + test | +| 2 | Doc: transform APIs are privileged; host grant required | +| 3 | Optional: single-transaction exclusive activate | +| 4 | Optional: return share materialization receipt to caller | + +## Attack scenarios checked (no exploit) + +- Cross-tenant document via search without grants → post-filter + tenant SQL. +- Embed table name injection → pattern reject. +- Promote another tenant’s generation → tenantId filter on transform_run; + config tenant mismatch check on promote. +- Share grant self-grant → skipped when peer === sharedBy. diff --git a/CHANGELOG.md b/CHANGELOG.md index f4b5a7c..ff86b28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Citation `open.type` is now `"memory"`. - **Breaking:** env var is `DATABASE_URL` (was `KNOWLEDGE_DATABASE_URL`); pass `memory.databaseUrl` on config as an alternative. No deprecated alias. +- `add` (plane + HTTP) returns `{ documentId, versionId }` so share-grant audit + and provenance can name the version that carried the write. - Interchange `defineTool` factories at `@corbits/memory/tools` (`memory_add`, `memory_search`, `memory_list`) — HTTP clients for mounted hub routes with install env `memoryBaseUrl` / `memoryTenantId` / `memoryAuthToken`. Declared @@ -50,14 +52,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 mini-ACL. Share sugar only mints tags. See `docs/AUTHZ-DOCUMENT-ACCESS.md`. - **Breaking:** Postgres baseline is two files (`0001_extensions` + `0002_memory_baseline`) with `access_tags` and no `visibility_*` columns. - Fresh installs only — drop/recreate the memory schema on existing DBs. + Fresh installs only — drop/recreate the `memory` schema on existing DBs. - **Breaking:** `grantStore` + `conditionRegistry` are top-level `createMemory` options (no nested `grants: { … }`). ### Added - Optional `TextExtractor` + `file` XOR `content` on `add` -- `share` sugar on `add` (maps to access tags only: owner, tenant, peers) +- `share` sugar on `add` (maps to access tags; principals also materialize + grants when the host grant store is writable) - `access_tags` on `memory.document` (baseline schema) ### Removed diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md index 88cbb40..8a3762b 100644 --- a/IMPLEMENTATION.md +++ b/IMPLEMENTATION.md @@ -295,7 +295,7 @@ at activation instead. The dense query's `ORDER BY` is generated by `cosineDistanceExpr` from the same module so it always matches the indexed expression. The table name is validated against `EMBED_TABLE_NAME_PATTERN` -(`/^memory_embedding_[a-f0-9]{16}$/`) both when computed and again every +(`/^"memory"\."embedding_[a-f0-9]{16}"$/`) both when computed and again every time it's read back from `memory_embed_model`, before ever being string-interpolated into raw SQL — this is the only place in the codebase a computed identifier is spliced into DDL/DML. @@ -531,6 +531,12 @@ Plane methods (engine DocumentStore only): `createTransformConfig`, `listTransformConfigs`, `runTransform`, `promoteGeneration`, `demoteGeneration` on `Memory`. Custom/fake stores omit these methods. +**Host privilege:** transform/promote/demote take `tenantId` (and config/run +ids) only — no principal, no HTTP routes. They rewrite live generation and the +active dense embed model for a tenant. The host must treat them as admin +operations (grant-gate before calling); never expose them unauthenticated to +end-user agents. + ## Mounted routes @@ -544,7 +550,7 @@ Each route is guarded with `grantGuard(deps, action)`, which applies the host's | Method + path | Grant action | Request body | Response | |---|---|---|---| -| `POST /api/tenants/:tenantId/memory/add` | `add` | `{ title, text, access_tags?, share? }` | `200 { documentId }`; `400` on validation | +| `POST /api/tenants/:tenantId/memory/add` | `add` | `{ title, text, access_tags?, share? }` | `200 { documentId, versionId }`; `400` on validation | | `POST /api/tenants/:tenantId/memory/search` | `search` | `{ query, limit?, kinds?, entity_ids?, sources?, includeEvidence? }` (limit 1–50; `kinds`/`entity_ids`/`sources` narrow retrieval before fusion; unset or `[]` = unfiltered; `includeEvidence` adds a short evidence string when true) | `200 { items[], evidence?, degraded? }`; `400` on bad input | | `GET /api/tenants/:tenantId/memory/list` | `search` | query `?limit=` (1–100, string on the wire) | `200 { events: [{ at, title, source, tenantId, principalId }] }` — durable recent documents for the caller's scope, filtered with grant-tag access (`canAccessDocument`). One event per document (active live version). | diff --git a/src/services/search.test.ts b/src/services/search.test.ts index 942263f..5aca162 100644 --- a/src/services/search.test.ts +++ b/src/services/search.test.ts @@ -233,8 +233,7 @@ describe("fetchDenseCandidates hnsw tuning", () => { unsafe: (sqlText: string) => { statements.push(sqlText); return Promise.resolve( - sqlText.includes('FROM "memory"."embed_model"') || - sqlText.includes("FROM memory_embed_model") + sqlText.includes('FROM "memory"."embed_model"') ? [MODEL_ROW] : [], ); @@ -376,6 +375,7 @@ describe("fetchDenseCandidates kind/entity filtering", () => { unsafe: (sqlText: string, params?: unknown[]) => Promise; savepoint: (fn: (sp: FakeTx) => Promise) => Promise; }; + const statements: string[] = []; function evaluate(sqlText: string, params: unknown[]): unknown[] { let rows = DENSE_ROWS; const kindMatch = sqlText.match(/kd\.kind = ANY\(\$(\d+)/); @@ -396,6 +396,7 @@ describe("fetchDenseCandidates kind/entity filtering", () => { } const tx: FakeTx = { unsafe: (sqlText: string, params: unknown[] = []) => { + statements.push(sqlText); if (sqlText.includes("ORDER BY")) { return Promise.resolve(evaluate(sqlText, params)); } @@ -404,18 +405,21 @@ describe("fetchDenseCandidates kind/entity filtering", () => { savepoint: (fn: (sp: FakeTx) => Promise) => fn(tx), }; const rawSql = { - unsafe: (sqlText: string) => - Promise.resolve( - // CL-5233 qualified the table; keep the pre-qualify form so an - // accidental revert still fails this suite the same way. - sqlText.includes('FROM "memory"."embed_model"') || - sqlText.includes("FROM memory_embed_model") + unsafe: (sqlText: string) => { + statements.push(sqlText); + return Promise.resolve( + // CL-5233 qualified the table — only the fully-qualified form matches. + sqlText.includes('FROM "memory"."embed_model"') ? [MODEL_ROW] : [], - ), + ); + }, begin: (cb: (t: FakeTx) => Promise) => cb(tx), }; - return rawSql as unknown as Parameters[0]["sql"]; + return { + rawSql: rawSql as unknown as Parameters[0]["sql"], + statements, + }; } function baseArgs(sql: Parameters[0]["sql"]) { @@ -435,8 +439,9 @@ describe("fetchDenseCandidates kind/entity filtering", () => { } it("excludes a semantically-similar chunk whose document kind does not match `kinds`", async () => { + const fake = fakeRawSql(); const rows = await fetchDenseCandidates({ - ...baseArgs(fakeRawSql()), + ...baseArgs(fake.rawSql), kinds: ["task"], }); const chunkIds = rows?.map((r) => r.chunkId) ?? []; @@ -445,8 +450,9 @@ describe("fetchDenseCandidates kind/entity filtering", () => { }); it("excludes a semantically-similar chunk whose document is not linked to any requested entityId", async () => { + const fake = fakeRawSql(); const rows = await fetchDenseCandidates({ - ...baseArgs(fakeRawSql()), + ...baseArgs(fake.rawSql), entityIds: ["e-match"], }); const chunkIds = rows?.map((r) => r.chunkId) ?? []; @@ -454,16 +460,32 @@ describe("fetchDenseCandidates kind/entity filtering", () => { expect(chunkIds).not.toContain("chunk-note"); }); + it("entity filter targets memory.edge (not pre-rename knowledge_edge)", async () => { + const fake = fakeRawSql(); + await fetchDenseCandidates({ + ...baseArgs(fake.rawSql), + entityIds: ["e-match"], + }); + const denseSelect = fake.statements.find( + (s) => s.includes("ORDER BY") && s.includes("ke."), + ); + expect(denseSelect).toBeDefined(); + expect(denseSelect).toContain('FROM "memory"."edge" ke'); + expect(denseSelect).not.toContain("knowledge_edge"); + }); + it("applies no kind/entity predicate — and returns every semantically-similar chunk — when neither filter is provided", async () => { - const rows = await fetchDenseCandidates(baseArgs(fakeRawSql())); + const fake = fakeRawSql(); + const rows = await fetchDenseCandidates(baseArgs(fake.rawSql)); const chunkIds = rows?.map((r) => r.chunkId) ?? []; expect(chunkIds).toContain("chunk-task"); expect(chunkIds).toContain("chunk-note"); }); it("treats an empty kinds/entityIds array as no filter, same as lexical", async () => { + const fake = fakeRawSql(); const rows = await fetchDenseCandidates({ - ...baseArgs(fakeRawSql()), + ...baseArgs(fake.rawSql), kinds: [], entityIds: [], }); diff --git a/src/services/search.ts b/src/services/search.ts index 67f3037..700fdb3 100644 --- a/src/services/search.ts +++ b/src/services/search.ts @@ -534,7 +534,7 @@ export async function fetchDenseCandidates( if (entityIds && entityIds.length > 0) { params.push(entityIds); entityClause = `AND kd.id IN ( - SELECT ke.from_ref FROM memory_edge ke + SELECT ke.from_ref FROM "memory"."edge" ke WHERE ke.tenant_id = $1 AND ke.from_type = 'document' AND ke.to_type = 'entity' AND ke.to_ref = ANY($${params.length}::text[]) )`; From da50f80c673b99c62649bcaf7057c0e1a2ddd67d Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 10 Aug 2026 10:39:28 -0700 Subject: [PATCH 08/19] feat: Level 3 corroboration + capture feed (CL-5867/5868) Wire living relevancy from supports/contradicts edges into hybrid search ranking and evidence:strong gating. Add cursor-based capture feed (feed_seq, memory.feed, GET .../memory/feed) for the resident distiller. Also single-statement exclusive embed activation, grantsMaterialized on share add, light search attribution, and substrate docs. --- CHANGELOG.md | 10 +- IMPLEMENTATION.md | 14 ++- PRODUCT.md | 1 + docs/DISTILLER.md | 34 +++++++ docs/FEED.md | 37 +++++++ docs/RELEVANCY.md | 44 +++++++++ docs/RETENTION.md | 25 +++++ migrations/0006_capture_feed.sql | 9 ++ src/core/corroboration.test.ts | 105 ++++++++++++++++++++ src/core/corroboration.ts | 88 +++++++++++++++++ src/core/embed-model-registry.ts | 19 ++-- src/db/schema.ts | 5 + src/http-bodies.ts | 42 ++++++++ src/index.ts | 26 +++++ src/memory.ts | 144 +++++++++++++++++++++++++++- src/ports/types.ts | 34 +++++++ src/routes/feed.ts | 84 ++++++++++++++++ src/routes/mount.ts | 4 +- src/services/feed.test.ts | 23 +++++ src/services/feed.ts | 132 +++++++++++++++++++++++++ src/services/search.test.ts | 46 +++++++-- src/services/search.ts | 160 +++++++++++++++++++++++++++---- 22 files changed, 1043 insertions(+), 43 deletions(-) create mode 100644 docs/DISTILLER.md create mode 100644 docs/FEED.md create mode 100644 docs/RELEVANCY.md create mode 100644 docs/RETENTION.md create mode 100644 migrations/0006_capture_feed.sql create mode 100644 src/core/corroboration.test.ts create mode 100644 src/core/corroboration.ts create mode 100644 src/routes/feed.ts create mode 100644 src/services/feed.test.ts create mode 100644 src/services/feed.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index ff86b28..f29689f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,8 +60,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Optional `TextExtractor` + `file` XOR `content` on `add` - `share` sugar on `add` (maps to access tags; principals also materialize - grants when the host grant store is writable) + grants when the host grant store is writable); `grantsMaterialized` on the + add result when peers were requested - `access_tags` on `memory.document` (baseline schema) +- Claim-bearing schema, temporal model, transform/replay, share grants + (resident memory distillation foundation — CL-5865/5866/5872/5873) +- Living relevancy: corroboration factor from supports/contradicts edges; + strong evidence gate (CL-5867). See `docs/RELEVANCY.md` +- Capture feed: `memory.feed` + `GET .../memory/feed` with `feed_seq` cursor + (CL-5868). See `docs/FEED.md` +- Search items include optional additive `attribution` (versionId, createdByKind, …) ### Removed diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md index 8a3762b..2b37c45 100644 --- a/IMPLEMENTATION.md +++ b/IMPLEMENTATION.md @@ -22,7 +22,7 @@ src/ mount.ts # registerMemoryRoutes (HTTP) deps.ts # RouteDeps, caller(c) (context identity), grantGuard - add.ts, search.ts, list.ts + add.ts, search.ts, list.ts, feed.ts db/ schema.ts # Drizzle table defs (memory.* schema) client.ts # createDb(config) -> { db (drizzle), sql (raw postgres-js) } @@ -31,6 +31,8 @@ src/ search.ts # hybridSearch and every retrieval-candidate query timeline.ts # listTimelineEvents — durable recent docs + grant-tag filter transform.ts # transform_config CRUD + runTransform (replay) + feed.ts # capture feed cursor pull (CL-5868) + share-grants.ts # peer grant materialization on share (CL-5873) core/ # framework-agnostic (chunking, embed/rerank, merge, schemas) # DocumentStore adapters / tools live as sibling packages (not in this tree): # @corbits/mem0-memory-adapter → github.com/corbitsdev/corbits-mem0-memory-adapter @@ -403,6 +405,16 @@ Entry point, one query in, one ranked/citable hit list out. `k` is clamped to is only accepted if `kinds` or `entityIds` is provided (structured-filter-only search); otherwise it throws `MemorySearchInputError` (400). +**Living relevancy (CL-5867):** after fusion, `attachCorroborationCounts` loads +`supports`/`contradicts` edge counts per version. Ranking multiplies by +`corroborationFactor` (bounded [0.7, 1.3]); evidence:strong also requires the +gate in `core/corroboration.ts` (stated human **or** support count ≥ floor). +Capture-time `authority` is never rewritten. See `docs/RELEVANCY.md`. + +**Capture feed (CL-5868):** `memory.feed({ after, limit, excludeGenerator? })` +and `GET .../memory/feed` pull live versions ordered by `feed_seq` (migration +`0006_capture_feed.sql`). Grant-checked like search. See `docs/FEED.md`. + 1. **Generation resolution** — `generation` defaults to `LIVE_GENERATION`. For any non-live generation, `resolveGenerationSearchParams` (transform.ts) looks up the owning `transform_run` → `transform_config` **for the search diff --git a/PRODUCT.md b/PRODUCT.md index 94dee05..d29ebf3 100644 --- a/PRODUCT.md +++ b/PRODUCT.md @@ -27,6 +27,7 @@ never creates one; it mounts onto yours. | `add` | `POST /api/tenants/:tenantId/memory/add` | `memory:add` | Capture a document | | `search` | `POST /api/tenants/:tenantId/memory/search` | `memory:search` | Hybrid retrieval (+ optional live sources) | | `list` | `GET /api/tenants/:tenantId/memory/list` | `memory:search` | Recent documents for the principal | +| `feed` | `GET /api/tenants/:tenantId/memory/feed` | `memory:search` | Cursor pull of new live versions (distiller) | Identity is always **`principalId` + `tenantId`** on the plane. HTTP routes never take body identity — they read `c.get("principal")` from Interchange diff --git a/docs/DISTILLER.md b/docs/DISTILLER.md new file mode 100644 index 0000000..b1a7b3f --- /dev/null +++ b/docs/DISTILLER.md @@ -0,0 +1,34 @@ +# Resident distiller (CL-5869) + +The **resident distiller** is a host workflow (Interchange `onTrigger`), not a +process inside this package. Memory exposes the **substrate** only: + +| Substrate | Where | +| --- | --- | +| Capture feed (exactly-once cursor) | `memory.feed` / `GET .../memory/feed` — [FEED.md](./FEED.md) | +| Claim-bearing + provenance | version columns — claim-bearing schema | +| Temporal classes | [TEMPORAL.md](./TEMPORAL.md) | +| Corroboration / living relevancy | [RELEVANCY.md](./RELEVANCY.md) | +| Transform / staged replay | `runTransform` / promote / demote | +| Share grants | `share.principals` → WritableGrantStore | + +## Recommended body shape + +1. **Pull** — `feed({ after: cursor, excludeGenerator: "resident-distiller" })` +2. **Classify / gate** — host policy (action-authority, kinds, poison skip) +3. **Distill** — host LLM; write via `add` with `createdByKind: agent`, + `generatorAgentId: "resident-distiller"`, provenance `inferred` as needed +4. **Link** — supports/contradicts edges (host or future write API) +5. **Advance cursor** — store `nextCursor` only after successful handling + (or after fail-soft poison quarantine) + +Loop-safety: always pass `excludeGenerator` matching the distiller’s +`generatorAgentId` so the feed never re-delivers the distiller’s own writes. + +## Out of scope here + +- Deploying the workflow, model choice, grant manifest contents beyond tags +- Push outbox (phase 2 of the feed — still pull-only) +- Automatic edge minting on add + +See `dispatch/resident-memory-distillation/5a-distiller_workflow/plan.md`. diff --git a/docs/FEED.md b/docs/FEED.md new file mode 100644 index 0000000..e42d21b --- /dev/null +++ b/docs/FEED.md @@ -0,0 +1,37 @@ +# Capture feed + +Stateless, cursorable pull of new **versions** for the resident distiller +(CL-5868). + +## Phase 1 — pull (implemented) + +``` +memory.feed({ tenantId, principalId, after?, limit?, excludeGenerator? }) +``` + +| Field | Meaning | +|-------|---------| +| `after` | Last consumed `feedSeq` (exclusive). Omit/0 = from start. | +| `limit` | Page size (bounded). | +| `excludeGenerator` | Skip versions with this `generator_agent_id` (loop-safe). | + +- Ordered by `feed_seq` ascending (Postgres `bigserial` on `memory.version`). +- **Live generation only** — same rule as default search. +- Capability: `memory` / `search` (same as list/retrieve). +- Document access: grant-tag post-filter identical to search. + +Cursor storage is the **consumer's** job (workflow run state). + +HTTP: `GET /api/tenants/:tenantId/memory/feed?after=&limit=&exclude_generator=` + +## Phase 2 — push (design only) + +Post-commit outbox row keyed by `feed_seq` + host dispatcher that mails the +deployment address with version ids. **Not implemented** in core. Phase 1 +`feed_seq` is the ordering key so Phase 2 is additive. + +## Non-goals + +- In-core cron or push dispatcher +- Bypassing grant tags for “tenant brain” +- Including replay generations in the default feed diff --git a/docs/RELEVANCY.md b/docs/RELEVANCY.md new file mode 100644 index 0000000..f230778 --- /dev/null +++ b/docs/RELEVANCY.md @@ -0,0 +1,44 @@ +# Claim relevancy (corroboration) + +Living relevancy for claim-bearing versions. Capture-time **`authority`** remains +a frozen snapshot (`computeAuthority` at write). Search derives a separate +**corroboration factor** from graph edges and multiplies ranking with it. + +## Edges + +| `rel` | Effect | +|-------|--------| +| `supports` | Independent source backs the target claim version — factor up | +| `contradicts` | Disagreement signal — factor down; **no** auto-delete/supersede | + +Counts are edges with `to_type = 'version'` and `to_ref = `. +Writers (typically the resident distiller) attach `supports` / `contradicts` +hints on capture; core does **not** decide claim sameness. + +## Ranking + +`corroborationFactor({ supports, contradicts })` ∈ **[0.7, 1.3]** (same envelope +as authority/recency boosts). Neutral `1.0` when both counts are zero. + +Effective authority for rank priors: + +``` +effectiveAuthority = clamp01(captureAuthority × corroborationFactor) +``` + +## Evidence: strong + +After relevance floors clear, **strong** also requires: + +1. Capture authority ≥ `AUTHORITY_STRONG_FLOOR` (0.3), and +2. Either: + - `provenance: stated` + `created_by_kind: human`, or + - support count ≥ `CORROBORATION_STRONG_FLOOR` (default **2**) + +Constants live in `src/core/corroboration.ts`. + +## Non-goals + +- Embedding-similarity merge of claims +- Silent resolution of human disagreement +- Inference inside `@corbits/memory` diff --git a/docs/RETENTION.md b/docs/RETENTION.md new file mode 100644 index 0000000..519f9d4 --- /dev/null +++ b/docs/RETENTION.md @@ -0,0 +1,25 @@ +# Retention classes (CL-5871) + +**Status:** schema-ready design; write-path helpers land with 4b. + +Versions may carry a **retention class** orthogonal to temporal ranking class: + +| Class | Intent | +| --- | --- | +| `ephemeral` | Short TTL; auto-eligible for hard delete after window | +| `working` | Default working memory; soft deprecate then TTL | +| `durable` | Long-lived claims; deprecate/tombstone only on explicit write | +| `legal_hold` | Never auto-delete; operator-only release | + +## Write paths (planned) + +| Verb | Effect | +| --- | --- | +| `deprecate(versionId, reason)` | `status=deprecated`, set `deprecated_at` / reason | +| `tombstone(documentId)` | Hide from search/feed; retain row for audit | +| TTL sweeper | Host cron: hard-delete `ephemeral` past `valid_until` / retention window | + +Search and feed already exclude non-active (and non-superseded for feed) rows; +retention classes refine *when* those transitions fire, not ranking math. + +See `dispatch/resident-memory-distillation/4b-retention_forgetting/plan.md`. diff --git a/migrations/0006_capture_feed.sql b/migrations/0006_capture_feed.sql new file mode 100644 index 0000000..82c59d4 --- /dev/null +++ b/migrations/0006_capture_feed.sql @@ -0,0 +1,9 @@ +-- Capture feed: monotonic commit marker for exactly-once pull (CL-5868). +-- feed_seq doubles as the Phase-2 outbox ordering key (design only; no push). + +ALTER TABLE "memory"."version" + ADD COLUMN IF NOT EXISTS "feed_seq" bigserial; + +-- Live-generation drain path: tenant + generation + cursor. +CREATE INDEX IF NOT EXISTS "version_feed_seq_idx" + ON "memory"."version" ("tenant_id", "generation", "feed_seq"); diff --git a/src/core/corroboration.test.ts b/src/core/corroboration.test.ts new file mode 100644 index 0000000..3bcb7be --- /dev/null +++ b/src/core/corroboration.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from "bun:test"; +import { + corroborationFactor, + CORROBORATION_STRONG_FLOOR, + effectiveAuthority, + meetsStrongEvidenceGate, +} from "./corroboration.ts"; +import { + BOOST_MULTIPLIER_MAX, + BOOST_MULTIPLIER_MIN, +} from "./hybrid-search.ts"; + +describe("corroborationFactor", () => { + it("is neutral with no edges", () => { + expect(corroborationFactor({ supports: 0, contradicts: 0 })).toBe(1); + }); + + it("raises rank with independent supports", () => { + const none = corroborationFactor({ supports: 0, contradicts: 0 }); + const one = corroborationFactor({ supports: 1, contradicts: 0 }); + const many = corroborationFactor({ supports: 4, contradicts: 0 }); + expect(one).toBeGreaterThan(none); + expect(many).toBeGreaterThan(one); + expect(many).toBeLessThanOrEqual(BOOST_MULTIPLIER_MAX); + }); + + it("lowers rank with contradictions without zeroing", () => { + const base = corroborationFactor({ supports: 0, contradicts: 0 }); + const hit = corroborationFactor({ supports: 0, contradicts: 2 }); + expect(hit).toBeLessThan(base); + expect(hit).toBeGreaterThanOrEqual(BOOST_MULTIPLIER_MIN); + }); + + it("stays inside the boost envelope", () => { + for (const s of [0, 1, 2, 8, 100]) { + for (const c of [0, 1, 2, 8, 100]) { + const f = corroborationFactor({ supports: s, contradicts: c }); + expect(f).toBeGreaterThanOrEqual(BOOST_MULTIPLIER_MIN); + expect(f).toBeLessThanOrEqual(BOOST_MULTIPLIER_MAX); + } + } + }); +}); + +describe("effectiveAuthority", () => { + it("scales the capture snapshot without mutating the formula range", () => { + const snap = 0.8; + const raised = effectiveAuthority(snap, { supports: 4, contradicts: 0 }); + const lowered = effectiveAuthority(snap, { supports: 0, contradicts: 4 }); + expect(raised).toBeGreaterThan(snap * 0.99); + expect(lowered).toBeLessThan(snap); + expect(raised).toBeLessThanOrEqual(1); + expect(lowered).toBeGreaterThanOrEqual(0); + }); +}); + +describe("meetsStrongEvidenceGate", () => { + const floor = 0.3; + + it("rejects low authority even with supports", () => { + expect( + meetsStrongEvidenceGate({ + authority: 0.1, + supports: 10, + authorityFloor: floor, + }), + ).toBe(false); + }); + + it("accepts stated human above authority floor without supports", () => { + expect( + meetsStrongEvidenceGate({ + authority: 0.5, + supports: 0, + provenance: "stated", + createdByKind: "human", + authorityFloor: floor, + }), + ).toBe(true); + }); + + it("accepts corroboration at the strong floor", () => { + expect( + meetsStrongEvidenceGate({ + authority: 0.5, + supports: CORROBORATION_STRONG_FLOOR, + provenance: "inferred", + createdByKind: "agent", + authorityFloor: floor, + }), + ).toBe(true); + }); + + it("rejects inferred agent below corroboration floor", () => { + expect( + meetsStrongEvidenceGate({ + authority: 0.5, + supports: CORROBORATION_STRONG_FLOOR - 1, + provenance: "inferred", + createdByKind: "agent", + authorityFloor: floor, + }), + ).toBe(false); + }); +}); diff --git a/src/core/corroboration.ts b/src/core/corroboration.ts new file mode 100644 index 0000000..2c407bb --- /dev/null +++ b/src/core/corroboration.ts @@ -0,0 +1,88 @@ +/** + * Living claim relevancy from supports/contradicts edge counts. + * + * Capture-time `authority` stays a frozen snapshot. Search multiplies ranking + * by a bounded corroboration factor derived from graph edges (version targets). + * Core never decides claim sameness — the distiller chooses supports vs new write. + * + * See docs/RELEVANCY.md. + */ +import { + BOOST_MULTIPLIER_MAX, + BOOST_MULTIPLIER_MIN, + clampBoostMultiplier, +} from "./hybrid-search.ts"; + +/** Independent supports needed for evidence:strong (with authority floor). */ +export const CORROBORATION_STRONG_FLOOR = 2; + +/** Log-scale cap for support/contradict counts (mirrors actor-count plateau). */ +export const CORROBORATION_COUNT_LOG_CAP = 4; + +const BOOST_BASE = BOOST_MULTIPLIER_MIN; +const BOOST_SPAN = BOOST_MULTIPLIER_MAX - BOOST_MULTIPLIER_MIN; + +export type CorroborationCounts = { + supports: number; + contradicts: number; +}; + +/** + * Bounded multiplier in [0.7, 1.3]. Neutral (1.0) when no edges. + * Supports raise rank; contradicts lower it — never auto-delete. + */ +export function corroborationFactor(counts: CorroborationCounts): number { + const supports = Math.max(0, Math.floor(counts.supports)); + const contradicts = Math.max(0, Math.floor(counts.contradicts)); + if (supports === 0 && contradicts === 0) { + return 1; + } + const denom = Math.log(1 + CORROBORATION_COUNT_LOG_CAP); + const supportScore = + denom > 0 ? Math.min(1, Math.log(1 + supports) / denom) : 0; + const contradictScore = + denom > 0 ? Math.min(1, Math.log(1 + contradicts) / denom) : 0; + // Midpoint 0.5 + half support − half contradict → [0, 1] then map to envelope. + const unit = Math.min( + 1, + Math.max(0, 0.5 + 0.5 * supportScore - 0.5 * contradictScore), + ); + return clampBoostMultiplier(BOOST_BASE + BOOST_SPAN * unit); +} + +/** + * Effective authority for ranking: snapshot × corroboration factor, clamped to + * [0, 1] so authority-weighted formulas stay in range. + */ +export function effectiveAuthority( + captureAuthority: number, + counts: CorroborationCounts, +): number { + const factor = corroborationFactor(counts); + return Math.min(1, Math.max(0, captureAuthority * factor)); +} + +export type StrongEvidenceSignals = { + /** Capture-time authority (0..1). */ + authority: number; + /** Independent supports targeting this version. */ + supports: number; + provenance?: string | undefined; + createdByKind?: string | undefined; + /** Default AUTHORITY_STRONG_FLOOR from search. */ + authorityFloor: number; + corroborationFloor?: number | undefined; +}; + +/** + * Whether evidence may report strong given relevance already cleared. + * Requires authority floor AND (stated human OR supports ≥ floor). + */ +export function meetsStrongEvidenceGate(signals: StrongEvidenceSignals): boolean { + if (signals.authority < signals.authorityFloor) return false; + const floor = signals.corroborationFloor ?? CORROBORATION_STRONG_FLOOR; + const statedHuman = + signals.provenance === "stated" && signals.createdByKind === "human"; + if (statedHuman) return true; + return Math.max(0, Math.floor(signals.supports)) >= floor; +} diff --git a/src/core/embed-model-registry.ts b/src/core/embed-model-registry.ts index 47f05b9..10a04a0 100644 --- a/src/core/embed-model-registry.ts +++ b/src/core/embed-model-registry.ts @@ -305,20 +305,23 @@ export async function clearActiveEmbedModels( ); } -/** Make `modelKey` the sole active model for the tenant. */ +/** Make `modelKey` the sole active model for the tenant (single statement). */ async function setActiveEmbedModelExclusive( client: EmbedRegistrySqlClient, tenantId: string, modelKey: string, ): Promise { + // One round-trip: demote every other active row and activate the target. + // Concurrent activate still races at the app level; this removes the + // two-UPDATE window where zero or two actives can briefly exist. await client.query( - `UPDATE "memory"."embed_model" - SET status = 'ready', updated_at = now() - WHERE tenant_id = $1 AND status = 'active' AND model_key <> $2`, - [tenantId, modelKey], - ); - await client.query( - `UPDATE "memory"."embed_model" + `WITH demoted AS ( + UPDATE "memory"."embed_model" + SET status = 'ready', updated_at = now() + WHERE tenant_id = $1 AND status = 'active' AND model_key <> $2 + RETURNING 1 + ) + UPDATE "memory"."embed_model" SET status = 'active', updated_at = now() WHERE tenant_id = $1 AND model_key = $2`, [tenantId, modelKey], diff --git a/src/db/schema.ts b/src/db/schema.ts index 5337c67..9188606 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -1,4 +1,5 @@ import { + bigint, boolean, customType, integer, @@ -84,6 +85,9 @@ export const memoryVersion = memorySchema.table( // version it writes with its own transform_run id instead, so a replayed // corpus never collides with (or supersedes) the live one. generation: text("generation").notNull().default("live"), + // Monotonic commit marker for capture-feed pull (CL-5868). Assigned by + // Postgres bigserial (see migrations/0006_capture_feed.sql); not set in app. + feedSeq: bigint("feed_seq", { mode: "number" }), }, (t) => [ uniqueIndex("version_document_generation_version_uniq").on( @@ -92,6 +96,7 @@ export const memoryVersion = memorySchema.table( t.version, ), index("version_document_status_idx").on(t.documentId, t.status), + index("version_feed_seq_idx").on(t.tenantId, t.generation, t.feedSeq), ], ); diff --git a/src/http-bodies.ts b/src/http-bodies.ts index 716a957..93b8e0e 100644 --- a/src/http-bodies.ts +++ b/src/http-bodies.ts @@ -73,6 +73,48 @@ export function parseListLimitString( return n; } +/** HTTP query schema for GET /memory/feed. */ +export const FeedQuery = type({ + "after?": "string", + "limit?": "string", + "exclude_generator?": "string", +}); + +export type FeedQuery = typeof FeedQuery.infer; + +export type ParsedFeedQuery = { + after?: number; + limit?: number; + excludeGenerator?: string; +}; + +/** + * Parse feed query params. Returns `{ ok: false, error }` on invalid numbers. + */ +export function parseFeedQuery( + q: FeedQuery, +): { ok: true; value: ParsedFeedQuery } | { ok: false; error: string } { + const value: ParsedFeedQuery = {}; + if (q.after !== undefined && q.after !== "") { + const n = Number(q.after); + if (!Number.isInteger(n) || n < 0) { + return { ok: false, error: "after must be a non-negative integer" }; + } + value.after = n; + } + if (q.limit !== undefined && q.limit !== "") { + const n = Number(q.limit); + if (!Number.isInteger(n) || n < 1 || n > 100) { + return { ok: false, error: "limit must be an integer from 1 to 100" }; + } + value.limit = n; + } + if (q.exclude_generator !== undefined && q.exclude_generator !== "") { + value.excludeGenerator = q.exclude_generator; + } + return { ok: true, value }; +} + /** Coerce LLM-stringified integers before arktype number.integer checks. */ export function coerceOptionalLimitArg( args: Record, diff --git a/src/index.ts b/src/index.ts index e0991fa..1312ba6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -40,9 +40,13 @@ export type { Memory, MemoryOptions, MemoryListParams, + MemoryFeedParams, + MemoryFeedEntry, + MemoryFeedResult, MemoryShare, SearchHit, SearchItem, + SearchAttribution, SearchResult, TextExtractor, TimelineEvent, @@ -111,6 +115,28 @@ export { type TransformRunRow, } from "./services/transform.ts"; +// Capture feed (CL-5868) +export { + fetchFeed, + FEED_LIMIT_DEFAULT, + FEED_LIMIT_MAX, + FEED_LIMIT_MIN, + type FeedArgs, + type FeedEntry, + type FeedResult, +} from "./services/feed.ts"; + +// Corroboration / living relevancy (CL-5867) +export { + corroborationFactor, + CORROBORATION_COUNT_LOG_CAP, + CORROBORATION_STRONG_FLOOR, + effectiveAuthority, + meetsStrongEvidenceGate, + type CorroborationCounts, + type StrongEvidenceSignals, +} from "./core/corroboration.ts"; + // Embed model registry (ensure vs activate) export { activateEmbedModel, diff --git a/src/memory.ts b/src/memory.ts index 64b23db..153e17d 100644 --- a/src/memory.ts +++ b/src/memory.ts @@ -26,6 +26,10 @@ import { listTimelineEvents, type TimelineEvent, } from "./services/timeline.ts"; +import { + fetchFeed, + type FeedEntry, +} from "./services/feed.ts"; import { createTransformConfig, demoteGeneration, @@ -163,7 +167,26 @@ export type MemoryAddParams = MemoryIdentity & { attributes?: Record; }; -export type MemoryAddResult = { documentId: string; versionId: string }; +export type MemoryAddResult = { + documentId: string; + versionId: string; + /** + * When `share.principals` was non-empty: true if peer grants were written + * to a WritableGrantStore; false if materialization was skipped (no writable + * store or soft failure). Omitted when no peer share was requested. + */ + grantsMaterialized?: boolean; +}; + +export type SearchAttribution = { + versionId: string; + provenance?: string; + createdByKind?: string; + generatorAgentId?: string | null; + evidence?: "strong" | "weak" | "none"; + supports?: number; + contradicts?: number; +}; export type SearchItem = { documentId: string; @@ -174,6 +197,8 @@ export type SearchItem = { citation: SearchHit["citation"]; /** ISO timestamp for merge recency when the store provides it. */ updatedAt?: string; + /** Additive provenance / temporal / corroboration for render-time attribution. */ + attribution?: SearchAttribution; }; export type SearchResult = { @@ -187,6 +212,32 @@ export type MemoryListParams = MemoryIdentity & { limit?: number; }; +export type MemoryFeedParams = MemoryIdentity & { + /** Exclusive cursor (last seen feedSeq). Default 0. */ + after?: number; + limit?: number; + excludeGenerator?: string; +}; + +export type MemoryFeedEntry = { + feedSeq: number; + versionId: string; + documentId: string; + kind: string; + title: string; + status: string; + createdByKind: string; + generatorAgentId: string | null; + provenance: string; + occurredAt: string; + createdAt: string; +}; + +export type MemoryFeedResult = { + entries: MemoryFeedEntry[]; + nextCursor: number | null; +}; + export class MemoryError extends Error { constructor( public readonly status: number, @@ -201,6 +252,11 @@ export type Memory = { search(params: MemorySearchParams): Promise; add(params: MemoryAddParams): Promise; list(params: MemoryListParams): Promise; + /** + * Cursor pull of new live versions (engine store only). Grant-checked like + * search. See docs/FEED.md. + */ + feed?(params: MemoryFeedParams): Promise; close(): Promise; /** * Transform / replay surface (engine DocumentStore only). Present when the @@ -315,6 +371,13 @@ function hitsToSearchItems(hits: readonly SearchHit[]): SearchItem[] { score: h.score, kind: h.kind, citation: h.citation, + attribution: { + versionId: h.version_id, + createdByKind: h.created_by_kind, + ...(h.generator_agent_id !== undefined + ? { generatorAgentId: h.generator_agent_id } + : {}), + }, })); } @@ -701,8 +764,10 @@ function createPlaneFromStore( // Share materialization (CL-5873): stamp document-scoped tag + write // peer grants when the host store is writable. Tag mint alone is not // enough for peers without host bootstrap grants on owner tags. + let grantsMaterialized: boolean | undefined; const peers = params.share?.principals; if (peers && peers.length > 0) { + grantsMaterialized = false; const docTag = documentTag(result.documentId); if (store.appendAccessTags) { await store.appendAccessTags(params.tenantId, result.documentId, [docTag]); @@ -720,6 +785,7 @@ function createPlaneFromStore( sourceVersionId: result.versionId, share: params.share ?? {}, }); + grantsMaterialized = true; } else { log.warn( "memory.add: share.principals set without WritableGrantStore; tags only (peers need host grants)", @@ -728,7 +794,9 @@ function createPlaneFromStore( } } - return result; + return grantsMaterialized === undefined + ? result + : { ...result, grantsMaterialized }; }, async list(params) { @@ -744,6 +812,28 @@ function createPlaneFromStore( }); }, + async feed(params) { + if (!store.feed) { + throw new MemoryError( + 501, + "feed requires the engine DocumentStore", + ); + } + return store.feed({ + tenantId: params.tenantId, + principalId: params.principalId, + ...(params.after !== undefined ? { after: params.after } : {}), + ...(params.limit !== undefined ? { limit: params.limit } : {}), + ...(params.excludeGenerator !== undefined + ? { excludeGenerator: params.excludeGenerator } + : {}), + ...(grants !== undefined ? { grants: grants.grantStore } : {}), + ...(grants?.conditionRegistry !== undefined + ? { conditionRegistry: grants.conditionRegistry } + : {}), + }); + }, + async close() { await store.close(); }, @@ -1054,6 +1144,56 @@ function createEngineDocumentStore(config: MemoryConfig): { }); }, + async feed(params) { + await ensureVerified(); + const raw = await fetchFeed(db, { + tenantId: params.tenantId, + ...(params.after !== undefined ? { after: params.after } : {}), + ...(params.limit !== undefined ? { limit: params.limit } : {}), + ...(params.excludeGenerator !== undefined + ? { excludeGenerator: params.excludeGenerator } + : {}), + }); + + const allowed: FeedEntry[] = []; + for (const entry of raw.entries) { + if (!params.grants) { + if (entry.createdByPrincipalId === params.principalId) { + allowed.push(entry); + } + continue; + } + const ok = await canAccessDocument({ + grants: params.grants, + tenantId: params.tenantId, + principalId: params.principalId, + createdByPrincipalId: entry.createdByPrincipalId, + accessTags: entry.accessTags, + ...(params.conditionRegistry !== undefined + ? { conditionRegistry: params.conditionRegistry } + : {}), + }); + if (ok) allowed.push(entry); + } + + const entries = allowed.map((e) => ({ + feedSeq: e.feedSeq, + versionId: e.versionId, + documentId: e.documentId, + kind: e.kind, + title: e.title, + status: e.status, + createdByKind: e.createdByKind, + generatorAgentId: e.generatorAgentId, + provenance: e.provenance, + occurredAt: e.occurredAt, + createdAt: e.createdAt, + })); + const nextCursor = + entries.length > 0 ? entries[entries.length - 1]!.feedSeq : null; + return { entries, nextCursor }; + }, + async close() { await sql.end({ timeout: 5 }); }, diff --git a/src/ports/types.ts b/src/ports/types.ts index 8ef1d08..5820d28 100644 --- a/src/ports/types.ts +++ b/src/ports/types.ts @@ -90,6 +90,35 @@ export type DocumentStoreListEvent = { principalId: string; }; +export type DocumentStoreFeedParams = { + tenantId: string; + principalId: string; + after?: number; + limit?: number; + excludeGenerator?: string; + grants?: GrantStore; + conditionRegistry?: ConditionRegistry; +}; + +export type DocumentStoreFeedEntry = { + feedSeq: number; + versionId: string; + documentId: string; + kind: string; + title: string; + status: string; + createdByKind: string; + generatorAgentId: string | null; + provenance: string; + occurredAt: string; + createdAt: string; +}; + +export type DocumentStoreFeedResult = { + entries: DocumentStoreFeedEntry[]; + nextCursor: number | null; +}; + /** * Durable document plane. Default implementation is the engine's pgvector * store. Hosts inject a DocumentStore (or fakes) via `options.documentStore` @@ -106,6 +135,11 @@ export type DocumentStore = { add(params: DocumentStoreAddParams): Promise; search(params: DocumentStoreSearchParams): Promise; list(params: DocumentStoreListParams): Promise; + /** + * Cursor pull of new live versions (CL-5868). Engine-only; vendor stores + * may omit (plane returns 501). + */ + feed?(params: DocumentStoreFeedParams): Promise; close(): Promise; /** * Append access tags after insert (used by share materialization to stamp diff --git a/src/routes/feed.ts b/src/routes/feed.ts new file mode 100644 index 0000000..a8fb88f --- /dev/null +++ b/src/routes/feed.ts @@ -0,0 +1,84 @@ +import type { Hono } from "hono"; +import type { TenantEnv } from "@intx/hub-api"; +import { describeRoute, resolver, validator } from "hono-openapi"; +import { type } from "arktype"; + +import { formatCaughtError, log } from "../log.ts"; +import { FeedQuery, parseFeedQuery } from "../http-bodies.ts"; +import { MemoryError } from "../memory.ts"; +import type { RouteDeps } from "./deps.ts"; +import { caller, grantGuard, requirePrincipal } from "./deps.ts"; + +const FeedResponse = type({ + entries: type({ + feedSeq: "number", + versionId: "string", + documentId: "string", + kind: "string", + title: "string", + status: "string", + createdByKind: "string", + generatorAgentId: "string|null", + provenance: "string", + occurredAt: "string", + createdAt: "string", + }).array(), + nextCursor: "number|null", +}); + +export function mountFeedRoute(app: Hono, deps: RouteDeps): void { + app.get( + "/api/tenants/:tenantId/memory/feed", + + describeRoute({ + tags: ["memory"], + summary: "Pull new live versions after a cursor (capture feed)", + responses: { + 200: { + description: "Ordered feed page", + content: { + "application/json": { schema: resolver(FeedResponse) }, + }, + }, + 400: { description: "Invalid query params" }, + 401: { description: "No principal on the request context" }, + 403: { description: "Missing the memory:search grant" }, + 501: { description: "Feed requires the engine DocumentStore" }, + 502: { description: "Feed query failed" }, + }, + }), + requirePrincipal(), + grantGuard(deps, "search"), + validator("query", FeedQuery), + async (c) => { + const { scopeId, subjectId } = caller(c); + const parsed = parseFeedQuery(c.req.valid("query")); + if (!parsed.ok) { + return c.json({ error: parsed.error }, 400); + } + if (!deps.memory.feed) { + return c.json({ error: "feed requires the engine DocumentStore" }, 501); + } + try { + const result = await deps.memory.feed({ + tenantId: scopeId, + principalId: subjectId, + ...parsed.value, + }); + return c.json(result); + } catch (err) { + if (err instanceof MemoryError) { + return c.json( + { error: err.message }, + err.status as 400 | 501, + ); + } + const errMessage = formatCaughtError(err); + log.error(`memory feed failed: ${errMessage}`, { + error: errMessage, + }); + return c.json({ error: "feed failed" }, 502); + } + }, + ); +} diff --git a/src/routes/mount.ts b/src/routes/mount.ts index 34d38ae..bc1a760 100644 --- a/src/routes/mount.ts +++ b/src/routes/mount.ts @@ -10,10 +10,11 @@ import type { RouteDeps } from "./deps.ts"; import { mountAddRoute } from "./add.ts"; import { mountSearchRoute } from "./search.ts"; import { mountListRoute } from "./list.ts"; +import { mountFeedRoute } from "./feed.ts"; export type { GrantConfig, RouteDeps } from "./deps.ts"; -/** HTTP JSON routes: add, search, list. */ +/** HTTP JSON routes: add, search, list, feed. */ export function registerMemoryRoutes( app: Hono, deps: RouteDeps, @@ -21,4 +22,5 @@ export function registerMemoryRoutes( mountAddRoute(app, deps); mountSearchRoute(app, deps); mountListRoute(app, deps); + mountFeedRoute(app, deps); } diff --git a/src/services/feed.test.ts b/src/services/feed.test.ts new file mode 100644 index 0000000..32cb8ed --- /dev/null +++ b/src/services/feed.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "bun:test"; +import { + FEED_LIMIT_DEFAULT, + FEED_LIMIT_MAX, + FEED_LIMIT_MIN, + FeedInputError, +} from "./feed.ts"; + +describe("feed constants", () => { + it("bounds page size", () => { + expect(FEED_LIMIT_MIN).toBe(1); + expect(FEED_LIMIT_DEFAULT).toBeLessThanOrEqual(FEED_LIMIT_MAX); + expect(FEED_LIMIT_MAX).toBe(100); + }); +}); + +describe("FeedInputError", () => { + it("is a 400-class error", () => { + const err = new FeedInputError("bad cursor"); + expect(err.status).toBe(400); + expect(err.message).toBe("bad cursor"); + }); +}); diff --git a/src/services/feed.ts b/src/services/feed.ts new file mode 100644 index 0000000..4e0650b --- /dev/null +++ b/src/services/feed.ts @@ -0,0 +1,132 @@ +/** + * Capture feed — cursor pull of new versions for the resident distiller (CL-5868). + * See docs/FEED.md. + */ +import { and, asc, eq, gt, isNull, ne, or, sql } from "drizzle-orm"; + +import type { Db } from "../db/client.ts"; +import { + memoryDocument, + memoryVersion, +} from "../db/schema.ts"; +import { LIVE_GENERATION } from "../core/generation.ts"; + +export const FEED_LIMIT_MIN = 1; +export const FEED_LIMIT_MAX = 100; +export const FEED_LIMIT_DEFAULT = 50; + +export type FeedArgs = { + tenantId: string; + /** Exclusive cursor: return rows with feed_seq > after. Default 0. */ + after?: number; + limit?: number; + /** Skip versions written by this generator (loop-safe for distiller). */ + excludeGenerator?: string; +}; + +export type FeedEntry = { + feedSeq: number; + versionId: string; + documentId: string; + kind: string; + title: string; + status: string; + createdByKind: string; + generatorAgentId: string | null; + provenance: string; + occurredAt: string; + createdAt: string; + /** Resource tags for grant post-filter (same as search). */ + accessTags: string[]; + createdByPrincipalId: string | null; +}; + +export type FeedResult = { + entries: FeedEntry[]; + /** Highest feedSeq in this page (consumer stores as next `after`). */ + nextCursor: number | null; +}; + +export class FeedInputError extends Error { + readonly status = 400; + constructor(message: string) { + super(message); + this.name = "FeedInputError"; + } +} + +export async function fetchFeed( + db: Db, + args: FeedArgs, +): Promise { + const after = Math.max(0, Math.floor(args.after ?? 0)); + const limit = Math.min( + FEED_LIMIT_MAX, + Math.max(FEED_LIMIT_MIN, Math.floor(args.limit ?? FEED_LIMIT_DEFAULT)), + ); + const exclude = args.excludeGenerator?.trim() || undefined; + + const conditions = [ + eq(memoryVersion.tenantId, args.tenantId), + eq(memoryVersion.generation, LIVE_GENERATION), + // Active (or superseded historical) live rows — exclude tombstones later in 4b. + // Feed surfaces commits; status filter keeps deprecated/tombstoned out once written. + sql`${memoryVersion.status} IN ('active', 'superseded')`, + gt(memoryVersion.feedSeq, after), + ]; + + if (exclude) { + conditions.push( + or( + isNull(memoryVersion.generatorAgentId), + ne(memoryVersion.generatorAgentId, exclude), + )!, + ); + } + + const rows = await db + .select({ + feedSeq: memoryVersion.feedSeq, + versionId: memoryVersion.id, + documentId: memoryVersion.documentId, + kind: memoryDocument.kind, + title: memoryDocument.title, + status: memoryVersion.status, + createdByKind: memoryVersion.createdByKind, + generatorAgentId: memoryVersion.generatorAgentId, + provenance: memoryVersion.provenance, + occurredAt: memoryVersion.occurredAt, + createdAt: memoryVersion.ingestedAt, + accessTags: memoryDocument.accessTags, + createdByPrincipalId: memoryVersion.createdByPrincipalId, + }) + .from(memoryVersion) + .innerJoin( + memoryDocument, + eq(memoryDocument.id, memoryVersion.documentId), + ) + .where(and(...conditions)) + .orderBy(asc(memoryVersion.feedSeq)) + .limit(limit); + + const entries: FeedEntry[] = rows.map((r) => ({ + feedSeq: Number(r.feedSeq), + versionId: r.versionId, + documentId: r.documentId, + kind: r.kind, + title: r.title, + status: r.status, + createdByKind: r.createdByKind, + generatorAgentId: r.generatorAgentId, + provenance: r.provenance, + occurredAt: r.occurredAt.toISOString(), + createdAt: r.createdAt.toISOString(), + accessTags: (r.accessTags as string[] | null) ?? [], + createdByPrincipalId: r.createdByPrincipalId, + })); + + const nextCursor = + entries.length > 0 ? entries[entries.length - 1]!.feedSeq : null; + + return { entries, nextCursor }; +} diff --git a/src/services/search.test.ts b/src/services/search.test.ts index 5aca162..af81e5f 100644 --- a/src/services/search.test.ts +++ b/src/services/search.test.ts @@ -134,25 +134,23 @@ describe("deriveHybridEvidence", () => { expect(deriveHybridEvidence(lexicalRows, 1)).toBe("weak"); }); - // The bug this fix closes: a query resolved mostly through the DENSE - // channel has a low (or zero) lexical ts_rank, so before this fix - // deriveHybridEvidence always fell through to deriveEvidence(lexicalRows) - // and reported "weak" — even when the cross-encoder rerank was highly - // confident and authority was high. The reranked-path floor now reports - // "strong" in that case instead. - it("reports 'strong' when reranked + high rerank score + high authority, even though lexical ts_rank is low", () => { + // Living relevancy (CL-5867): strong also needs the corroboration gate — + // stated human OR supports ≥ floor. High authority alone is not enough. + it("reports 'strong' when reranked + high rerank score + high authority + supports, even though lexical ts_rank is low", () => { const lowLexicalRows = [candidate({ rank: 0.001, authority: 0.9 })]; const evidence = deriveHybridEvidence(lowLexicalRows, 1, { rerankScore: 0.85, authority: 0.9, + supports: 2, }); expect(evidence).toBe("strong"); }); - it("reports 'strong' even with NO lexical rows at all, given a confident reranked top hit", () => { + it("reports 'strong' even with NO lexical rows at all, given a confident reranked top hit with supports", () => { const evidence = deriveHybridEvidence([], 1, { rerankScore: 0.85, authority: 0.9, + supports: 2, }); expect(evidence).toBe("strong"); }); @@ -161,6 +159,7 @@ describe("deriveHybridEvidence", () => { const evidence = deriveHybridEvidence([], 1, { rerankScore: 0.2, authority: 0.9, + supports: 5, }); expect(evidence).toBe("weak"); }); @@ -169,12 +168,41 @@ describe("deriveHybridEvidence", () => { const evidence = deriveHybridEvidence([], 1, { rerankScore: 0.9, authority: 0.1, + supports: 5, }); expect(evidence).toBe("weak"); }); + it("reports 'weak' when high score/authority but no corroboration gate (no supports, not stated human)", () => { + const evidence = deriveHybridEvidence([], 1, { + rerankScore: 0.9, + authority: 0.9, + supports: 0, + provenance: "inferred", + createdByKind: "agent", + }); + expect(evidence).toBe("weak"); + }); + + it("reports 'strong' for stated human without supports when score floors clear", () => { + const evidence = deriveHybridEvidence([], 1, { + rerankScore: 0.85, + authority: 0.9, + supports: 0, + provenance: "stated", + createdByKind: "human", + }); + expect(evidence).toBe("strong"); + }); + it("falls back to the lexical evidence path when reranking did not run (no rerankedTop)", () => { - const strongLexicalRows = [candidate({ rank: 0.9, authority: 0.9 })]; + const strongLexicalRows = [ + candidate({ + rank: 0.9, + authority: 0.9, + supports: 2, + }), + ]; expect(deriveHybridEvidence(strongLexicalRows, 1)).toBe("strong"); }); }); diff --git a/src/services/search.ts b/src/services/search.ts index 700fdb3..65cb58b 100644 --- a/src/services/search.ts +++ b/src/services/search.ts @@ -42,6 +42,12 @@ import { toRankedCandidates, type DegradeFlag, } from "../core/hybrid-search.ts"; +import { + corroborationFactor, + effectiveAuthority, + meetsStrongEvidenceGate, + type CorroborationCounts, +} from "../core/corroboration.ts"; import { formatCaughtError, log } from "../log.ts"; import { resolveGenerationSearchParams } from "./transform.ts"; import type { @@ -90,8 +96,9 @@ export function authorityWeightedScore( } // Evidence cap: a hit only reaches 'strong' when BOTH its raw lexical -// relevance clears STRONG_RANK_FLOOR AND its authority clears this floor. -const AUTHORITY_STRONG_FLOOR = 0.3; +// relevance clears STRONG_RANK_FLOOR AND the strong-evidence gate (authority +// floor + stated human OR corroboration floor) clears. See docs/RELEVANCY.md. +export const AUTHORITY_STRONG_FLOOR = 0.3; // Second, reranked-path-only strong floor. `deriveHybridEvidence`'s lexical // ts_rank check under-reports a query resolved mostly through the DENSE @@ -129,12 +136,17 @@ export interface CandidateRow { rank: number; occurredAt: Date; // The version's stored 0..1 authority score (computed at capture time; - // never recomputed here). + // never recomputed here). Ranking multiplies by corroborationFactor. authority: number; // Temporal ranking class + validity (docs/TEMPORAL.md). Defaults applied // when a row predates the temporal migration (should not happen after 0004). temporalClass: "event" | "deadline" | "state" | "lesson"; validUntil: Date | null; + /** Provenance mode for evidence gating (stated | inferred | unknown). */ + provenance?: string; + /** Independent supports / contradicts targeting this version (search-time). */ + supports?: number; + contradicts?: number; } export function snippet(text: string, maxLen = 240): string { @@ -190,7 +202,14 @@ export function toHit( status: row.status, score: scoreOverride ?? - authorityWeightedScore(row.rank, row.authority, authorityWeight), + authorityWeightedScore( + row.rank, + effectiveAuthority(row.authority, { + supports: row.supports ?? 0, + contradicts: row.contradicts ?? 0, + }), + authorityWeight, + ), title: row.title, snippet: snippet(row.snippetText), kind: row.kind, @@ -208,17 +227,24 @@ export function toHit( }; } -// Evidence is capped by authority, not just relevance: the top-ranked hit -// (by raw relevance) must ALSO clear AUTHORITY_STRONG_FLOOR to report -// 'strong'. A relevant hit backed only by a low-authority source reports -// 'weak' instead of overstating confidence. +// Evidence is capped by the strong-evidence gate (authority + corroboration / +// stated human), not relevance alone. A relevant hit with low authority or +// only inferred single-source content reports 'weak'. export function deriveEvidence( hits: readonly CandidateRow[], ): SearchResponse["evidence"] { if (hits.length === 0) return "none"; const top = hits.reduce((best, h) => (h.rank > best.rank ? h : best)); if (top.rank < STRONG_RANK_FLOOR) return "weak"; - return top.authority >= AUTHORITY_STRONG_FLOOR ? "strong" : "weak"; + return meetsStrongEvidenceGate({ + authority: top.authority, + supports: top.supports ?? 0, + provenance: top.provenance, + createdByKind: top.createdByKind, + authorityFloor: AUTHORITY_STRONG_FLOOR, + }) + ? "strong" + : "weak"; } // Evidence is primarily derived from the LEXICAL channel, using the same @@ -228,22 +254,34 @@ export function deriveEvidence( // against fused scores would make "strong" unreachable via that path. A // SECOND, independent "strong" path exists for the reranked path: when // `rerankedTop` is supplied (reranking ran and produced a top hit), a rerank -// score clearing RERANK_STRONG_FLOOR combined with authority clearing -// AUTHORITY_STRONG_FLOOR is strong evidence on its own, even when the -// lexical channel barely (or never) matched — this is what lets a query -// resolved mostly through the DENSE channel report "strong" instead of -// always "weak". A result that came back only through the dense channel, -// on the non-reranked/degraded path (no `rerankedTop`), is still "weak". +// score clearing RERANK_STRONG_FLOOR combined with the strong-evidence gate +// is strong evidence on its own, even when the lexical channel barely (or +// never) matched — this is what lets a query resolved mostly through the +// DENSE channel report "strong" instead of always "weak". A result that +// came back only through the dense channel, on the non-reranked/degraded +// path (no `rerankedTop`), is still "weak". export function deriveHybridEvidence( lexicalRows: readonly CandidateRow[], finalHitCount: number, - rerankedTop?: { rerankScore: number; authority: number }, + rerankedTop?: { + rerankScore: number; + authority: number; + supports?: number; + provenance?: string; + createdByKind?: string; + }, ): SearchResponse["evidence"] { if (finalHitCount === 0) return "none"; if ( rerankedTop && rerankedTop.rerankScore >= RERANK_STRONG_FLOOR && - rerankedTop.authority >= AUTHORITY_STRONG_FLOOR + meetsStrongEvidenceGate({ + authority: rerankedTop.authority, + supports: rerankedTop.supports ?? 0, + provenance: rerankedTop.provenance, + createdByKind: rerankedTop.createdByKind, + authorityFloor: AUTHORITY_STRONG_FLOOR, + }) ) { return "strong"; } @@ -266,7 +304,14 @@ export function dedupeCandidatesPerDocument( ): CandidateRow[] { const scoreOf = (row: CandidateRow): number => applyAuthorityPrior - ? authorityWeightedScore(row.rank, row.authority, authorityWeight) + ? authorityWeightedScore( + row.rank, + effectiveAuthority(row.authority, { + supports: row.supports ?? 0, + contradicts: row.contradicts ?? 0, + }), + authorityWeight, + ) : row.rank; const byDocument = new Map(); @@ -313,6 +358,50 @@ export async function attachEntityIds( return map; } +/** + * Batch-load supports/contradicts counts for candidate version ids. + * Edges target `to_type = version`. Mutates rows in place. + */ +export async function attachCorroborationCounts( + db: Db, + tenantId: string, + rows: CandidateRow[], +): Promise { + const versionIds = [...new Set(rows.map((r) => r.versionId))]; + if (versionIds.length === 0) return; + + const edges = await db + .select({ + toRef: knowledgeEdge.toRef, + rel: knowledgeEdge.rel, + }) + .from(knowledgeEdge) + .where( + and( + eq(knowledgeEdge.tenantId, tenantId), + eq(knowledgeEdge.toType, "version"), + inArray(knowledgeEdge.toRef, versionIds), + inArray(knowledgeEdge.rel, ["supports", "contradicts"]), + ), + ); + + const counts = new Map(); + for (const id of versionIds) { + counts.set(id, { supports: 0, contradicts: 0 }); + } + for (const edge of edges) { + const c = counts.get(edge.toRef) ?? { supports: 0, contradicts: 0 }; + if (edge.rel === "supports") c.supports += 1; + else if (edge.rel === "contradicts") c.contradicts += 1; + counts.set(edge.toRef, c); + } + for (const row of rows) { + const c = counts.get(row.versionId) ?? { supports: 0, contradicts: 0 }; + row.supports = c.supports; + row.contradicts = c.contradicts; + } +} + // The kinds/entityIds shape shared by both channels' candidate-query params // and by HybridSearchArgs (which fans a single caller-supplied pair of these // out to both channels before fusion). An empty array is treated identically @@ -408,6 +497,7 @@ export async function fetchLexicalCandidates( authority: memoryVersion.authority, temporalClass: memoryVersion.temporalClass, validUntil: memoryVersion.validUntil, + provenance: memoryVersion.provenance, }) .from(memoryChunk) .innerJoin( @@ -546,7 +636,8 @@ export async function fetchDenseCandidates( kd.adapter AS adapter, kd.external_ref AS external_ref, kv.created_by_kind AS created_by_kind, kv.generator_agent_id AS generator_agent_id, c.text AS snippet_text, kv.occurred_at AS occurred_at, kv.authority AS authority, - kv.temporal_class AS temporal_class, kv.valid_until AS valid_until + kv.temporal_class AS temporal_class, kv.valid_until AS valid_until, + kv.provenance AS provenance FROM ${activeTable.tableName} e JOIN "memory"."chunk" c ON c.id = e.chunk_id JOIN "memory"."version" kv ON kv.id = c.version_id @@ -612,6 +703,7 @@ export async function fetchDenseCandidates( validUntil: row["valid_until"] ? new Date(row["valid_until"] as string) : null, + provenance: (row["provenance"] as string | undefined) ?? "unknown", })); } @@ -685,7 +777,19 @@ function applyBoosts( const normalized = normalizeScoresToUnit(rows.map((row) => row.rank)); return rows.map((row, index) => { const normScore = normalized[index] ?? 0; - const authorityMult = authorityBoostMultiplier(row.authority); + const authorityMult = authorityBoostMultiplier( + effectiveAuthority(row.authority, { + supports: row.supports ?? 0, + contradicts: row.contradicts ?? 0, + }), + ); + // Corroboration also multiplies the final score once more via the same + // factor so a supported claim outranks an unsupported twin at equal + // authority snapshot (authorityBoost alone compresses high authorities). + const corrMult = corroborationFactor({ + supports: row.supports ?? 0, + contradicts: row.contradicts ?? 0, + }); const recencyMult = temporalRecencyMultiplier({ temporalClass: row.temporalClass, occurredAt: row.occurredAt, @@ -693,7 +797,10 @@ function applyBoosts( now, halfLifeMs: recencyHalfLifeMs, }); - return { row, finalScore: normScore * authorityMult * recencyMult }; + return { + row, + finalScore: normScore * authorityMult * corrMult * recencyMult, + }; }); } @@ -878,6 +985,12 @@ export async function hybridSearch( mergedRows.push({ ...base, rank: candidate.score }); } + // Living relevancy: attach supports/contradicts before authority-weighted + // dedupe / boosts (capture-time authority snapshot stays on the row). + await attachCorroborationCounts(db, tenantId, mergedRows); + // Lexical-only evidence path also needs counts on the original channel rows. + await attachCorroborationCounts(db, tenantId, lexicalRows); + let truncated: CandidateRow[]; // The final score per chunk, when the reranked path ran; absent on the // degraded/fallback path, where `toHit` falls back to its own @@ -1009,6 +1122,11 @@ export async function hybridSearch( ? { rerankScore: rawRerankScoreByChunk.get(topTruncated.chunkId) ?? 0, authority: topTruncated.authority, + supports: topTruncated.supports ?? 0, + ...(topTruncated.provenance !== undefined + ? { provenance: topTruncated.provenance } + : {}), + createdByKind: topTruncated.createdByKind, } : undefined; const evidence = deriveHybridEvidence(lexicalRows, hits.length, rerankedTop); From cce1fcbe6b2dc3087e390058795150e0c5c27834 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 10 Aug 2026 10:49:38 -0700 Subject: [PATCH 09/19] feat: wire attribution, retention, distiller substrate (CL-5870/5871/5869) Search hits surface additive provenance/temporal/corroboration/derived_from attribution. Retention classes + plane write paths (deprecate/tombstone/ hard-delete/sweep) land with migration 0007. Distiller remains host-side on feed+add; docs and CHANGELOG updated for Level 3 close-out. --- .gitignore | 3 + CHANGELOG.md | 9 +- IMPLEMENTATION.md | 12 ++ PRODUCT.md | 7 +- docs/DISTILLER.md | 10 +- docs/RETENTION.md | 38 +++--- migrations/0007_retention.sql | 16 +++ src/core/enums.lockstep.test.ts | 15 +++ src/core/enums.ts | 14 +- src/core/schemas/document.test.ts | 1 + src/core/schemas/document.ts | 8 ++ src/core/schemas/search.ts | 17 ++- src/db/schema.ts | 2 + src/http-bodies.ts | 1 + src/index.ts | 10 ++ src/memory.ts | 145 +++++++++++++++++++-- src/ports/types.ts | 20 +++ src/routes/search.ts | 16 ++- src/services/retention.test.ts | 23 ++++ src/services/retention.ts | 209 ++++++++++++++++++++++++++++++ src/services/search.ts | 162 +++++++++++++++++------ src/tools/search.ts | 8 ++ 22 files changed, 672 insertions(+), 74 deletions(-) create mode 100644 migrations/0007_retention.sql create mode 100644 src/services/retention.test.ts create mode 100644 src/services/retention.ts diff --git a/.gitignore b/.gitignore index d7f81b8..8dc025e 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ dispatch/ # Staging dirs for sibling package extracts (copy out with cp only) .staging-*/ + +# Local agent/session scratch — never commit +.corbits/ diff --git a/CHANGELOG.md b/CHANGELOG.md index f29689f..db69796 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,7 +69,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 strong evidence gate (CL-5867). See `docs/RELEVANCY.md` - Capture feed: `memory.feed` + `GET .../memory/feed` with `feed_seq` cursor (CL-5868). See `docs/FEED.md` -- Search items include optional additive `attribution` (versionId, createdByKind, …) +- **Wire attribution (CL-5870):** search hits carry additive `attribution` + (versionId, provenance, source/temporal class, createdByKind, + generatorAgentId, occurredAt/validUntil, corroboration counts, derivedFrom) +- **Retention (CL-5871):** `retention_class` on versions; plane APIs + `deprecateVersion` / `tombstoneDocument` / `hardDeleteDocument` / + `sweepEphemeral` / `setRetentionClass`; `includeDeprecated` on search. + See `docs/RETENTION.md` +- Distiller substrate notes: `docs/DISTILLER.md` (CL-5869 host workflow) ### Removed diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md index 2b37c45..3cca305 100644 --- a/IMPLEMENTATION.md +++ b/IMPLEMENTATION.md @@ -411,6 +411,18 @@ search); otherwise it throws `MemorySearchInputError` (400). gate in `core/corroboration.ts` (stated human **or** support count ≥ floor). Capture-time `authority` is never rewritten. See `docs/RELEVANCY.md`. +**Wire attribution (CL-5870):** `attachDerivedFrom` loads `derived_from` edges +onto candidates; `toHit` emits provenance, source/temporal class, occurred_at, +valid_until, corroboration counts, and derived_from. The plane maps these into +additive `SearchItem.attribution` (and `DocumentStoreSearchItem.attribution`). + +**Retention (CL-5871):** `memory.version.retention_class` + migration +`0007_retention.sql`. Plane helpers in `services/retention.ts` +(`deprecateVersion`, `tombstoneDocument`, `hardDeleteDocument`, +`sweepEphemeral`, `setRetentionClass`). Search accepts `includeDeprecated` +so lexical/dense can include `status IN ('active','deprecated')`. See +`docs/RETENTION.md`. + **Capture feed (CL-5868):** `memory.feed({ after, limit, excludeGenerator? })` and `GET .../memory/feed` pull live versions ordered by `feed_seq` (migration `0006_capture_feed.sql`). Grant-checked like search. See `docs/FEED.md`. diff --git a/PRODUCT.md b/PRODUCT.md index d29ebf3..743d388 100644 --- a/PRODUCT.md +++ b/PRODUCT.md @@ -25,10 +25,15 @@ never creates one; it mounts onto yours. | Method | HTTP | Grant | Meaning | | --- | --- | --- | --- | | `add` | `POST /api/tenants/:tenantId/memory/add` | `memory:add` | Capture a document | -| `search` | `POST /api/tenants/:tenantId/memory/search` | `memory:search` | Hybrid retrieval (+ optional live sources) | +| `search` | `POST /api/tenants/:tenantId/memory/search` | `memory:search` | Hybrid retrieval (+ optional live sources); hits may include additive `attribution` | | `list` | `GET /api/tenants/:tenantId/memory/list` | `memory:search` | Recent documents for the principal | | `feed` | `GET /api/tenants/:tenantId/memory/feed` | `memory:search` | Cursor pull of new live versions (distiller) | +Engine-only plane helpers (no HTTP yet): transform/replay, retention +(`deprecateVersion` / `tombstoneDocument` / … — see `docs/RETENTION.md`), +share-grant materialization. Distiller is a **host** `onTrigger` workflow +using feed + add + attribution (`docs/DISTILLER.md`). + Identity is always **`principalId` + `tenantId`** on the plane. HTTP routes never take body identity — they read `c.get("principal")` from Interchange context. diff --git a/docs/DISTILLER.md b/docs/DISTILLER.md index b1a7b3f..dcaea01 100644 --- a/docs/DISTILLER.md +++ b/docs/DISTILLER.md @@ -9,6 +9,8 @@ process inside this package. Memory exposes the **substrate** only: | Claim-bearing + provenance | version columns — claim-bearing schema | | Temporal classes | [TEMPORAL.md](./TEMPORAL.md) | | Corroboration / living relevancy | [RELEVANCY.md](./RELEVANCY.md) | +| Wire attribution on search | `SearchItem.attribution` (CL-5870) | +| Retention / forgetting | [RETENTION.md](./RETENTION.md) | | Transform / staged replay | `runTransform` / promote / demote | | Share grants | `share.principals` → WritableGrantStore | @@ -16,10 +18,12 @@ process inside this package. Memory exposes the **substrate** only: 1. **Pull** — `feed({ after: cursor, excludeGenerator: "resident-distiller" })` 2. **Classify / gate** — host policy (action-authority, kinds, poison skip) -3. **Distill** — host LLM; write via `add` with `createdByKind: agent`, +3. **Distill** — host LLM; write via `add` with agent identity and `generatorAgentId: "resident-distiller"`, provenance `inferred` as needed -4. **Link** — supports/contradicts edges (host or future write API) -5. **Advance cursor** — store `nextCursor` only after successful handling +4. **Link** — supports/contradicts edges (host write path) +5. **Attribute consumers** — search hits surface provenance, temporal class, + corroboration counts, and `derivedFrom` version ids for UI/citation +6. **Advance cursor** — store `nextCursor` only after successful handling (or after fail-soft poison quarantine) Loop-safety: always pass `excludeGenerator` matching the distiller’s diff --git a/docs/RETENTION.md b/docs/RETENTION.md index 519f9d4..6d3d86e 100644 --- a/docs/RETENTION.md +++ b/docs/RETENTION.md @@ -1,25 +1,31 @@ # Retention classes (CL-5871) -**Status:** schema-ready design; write-path helpers land with 4b. - -Versions may carry a **retention class** orthogonal to temporal ranking class: +Versions carry a **retention class** orthogonal to temporal ranking class +(`temporal_class`) and lineage (`source_class` / `provenance`). | Class | Intent | | --- | --- | -| `ephemeral` | Short TTL; auto-eligible for hard delete after window | -| `working` | Default working memory; soft deprecate then TTL | -| `durable` | Long-lived claims; deprecate/tombstone only on explicit write | -| `legal_hold` | Never auto-delete; operator-only release | +| `durable` | Long-lived claims; hard-delete blocked until tombstoned | +| `standard` | Default working memory | +| `ephemeral` | Short TTL; sweeper hard-deletes past `valid_until` (or 7d from `ingested_at`) | +| `source_only` | Keep raw capture; derived versions may be dropped by host policy | -## Write paths (planned) +Schema: `memory.version.retention_class` (migration `0007_retention.sql`). +CHECK constraint `version_retention_class_check` stays lockstep with +`RETENTION_CLASSES` in `src/core/enums.ts`. -| Verb | Effect | -| --- | --- | -| `deprecate(versionId, reason)` | `status=deprecated`, set `deprecated_at` / reason | -| `tombstone(documentId)` | Hide from search/feed; retain row for audit | -| TTL sweeper | Host cron: hard-delete `ephemeral` past `valid_until` / retention window | +## Write paths + +| Verb | Plane API | Effect | +| --- | --- | --- | +| Deprecate | `memory.deprecateVersion` | `status=deprecated`, `deprecated_at` / reason | +| Tombstone | `memory.tombstoneDocument` | All active/deprecated/superseded versions → `tombstoned`; chunk text redacted to `[redacted]` | +| Hard delete | `memory.hardDeleteDocument` | Deletes document row (cascade); **refuses** if any non-tombstoned version is `durable` | +| Sweep | `memory.sweepEphemeral` | Hard-deletes documents whose ephemeral versions are past TTL | +| Set class | `memory.setRetentionClass` | Update `retention_class` on a version | -Search and feed already exclude non-active (and non-superseded for feed) rows; -retention classes refine *when* those transitions fire, not ranking math. +Search and feed exclude non-active (and non-superseded for feed) rows by +default. Pass `includeDeprecated: true` on search to retrieve deprecated +versions intentionally (ops / audit). -See `dispatch/resident-memory-distillation/4b-retention_forgetting/plan.md`. +Service module: `src/services/retention.ts`. diff --git a/migrations/0007_retention.sql b/migrations/0007_retention.sql new file mode 100644 index 0000000..4366c4a --- /dev/null +++ b/migrations/0007_retention.sql @@ -0,0 +1,16 @@ +-- Retention classes + lifecycle write paths (CL-5871). +-- Orthogonal to temporal_class (ranking) and status (active/deprecated/…). + +ALTER TABLE "memory"."version" + ADD COLUMN IF NOT EXISTS "retention_class" text NOT NULL DEFAULT 'standard'; + +ALTER TABLE "memory"."version" + DROP CONSTRAINT IF EXISTS "version_retention_class_check"; + +ALTER TABLE "memory"."version" + ADD CONSTRAINT "version_retention_class_check" + CHECK ("retention_class" IN ('durable', 'standard', 'ephemeral', 'source_only')); + +CREATE INDEX IF NOT EXISTS "version_retention_ephemeral_idx" + ON "memory"."version" ("tenant_id", "retention_class", "valid_until") + WHERE "retention_class" = 'ephemeral'; diff --git a/src/core/enums.lockstep.test.ts b/src/core/enums.lockstep.test.ts index 9a9a5dc..6c3708e 100644 --- a/src/core/enums.lockstep.test.ts +++ b/src/core/enums.lockstep.test.ts @@ -6,12 +6,14 @@ import { EDGE_REF_TYPES_DB, LINEAGE_CLASSES, PROVENANCE_MODES, + RETENTION_CLASSES, TEMPORAL_CLASSES, } from "./enums.ts"; import { MemoryEdgeRelSchema, MemoryEdgeRefTypeSchema } from "./schemas/entity-edge.ts"; import { LineageClassSchema, ProvenanceModeSchema, + RetentionClassSchema, TemporalClassSchema, } from "./schemas/document.ts"; import { type } from "arktype"; @@ -86,6 +88,12 @@ describe("enum lockstep: TS constants match migration CHECK constraints", () => sorted(TEMPORAL_CLASSES), ); }); + + it("version_retention_class_check matches RETENTION_CLASSES", () => { + expect(sorted(lastCheckInList(sql, "version_retention_class_check"))).toEqual( + sorted(RETENTION_CLASSES), + ); + }); }); describe("enum lockstep: arktype accepts every SSOT value and rejects unknown", () => { @@ -126,4 +134,11 @@ describe("enum lockstep: arktype accepts every SSOT value and rejects unknown", } expect(TemporalClassSchema("forecast") instanceof type.errors).toBe(true); }); + + it("RetentionClassSchema accepts RETENTION_CLASSES only", () => { + for (const r of RETENTION_CLASSES) { + expect(RetentionClassSchema(r) instanceof type.errors).toBe(false); + } + expect(RetentionClassSchema("forever") instanceof type.errors).toBe(true); + }); }); diff --git a/src/core/enums.ts b/src/core/enums.ts index b7b6ff8..dc2399c 100644 --- a/src/core/enums.ts +++ b/src/core/enums.ts @@ -51,7 +51,7 @@ export const PROVENANCE_MODES = ["stated", "inferred", "unknown"] as const; export type ProvenanceMode = (typeof PROVENANCE_MODES)[number]; /** - * Temporal ranking class stored on knowledge.version.temporal_class. + * Temporal ranking class stored on memory.version.temporal_class. * See docs/TEMPORAL.md. */ export const TEMPORAL_CLASSES = [ @@ -62,6 +62,18 @@ export const TEMPORAL_CLASSES = [ ] as const; export type TemporalClass = (typeof TEMPORAL_CLASSES)[number]; +/** + * Retention class on memory.version.retention_class (CL-5871). + * Orthogonal to temporal_class (ranking) and status (lifecycle). + */ +export const RETENTION_CLASSES = [ + "durable", + "standard", + "ephemeral", + "source_only", +] as const; +export type RetentionClass = (typeof RETENTION_CLASSES)[number]; + /** Build an arktype union string from a const string array. */ export function arktypeStringUnion( values: readonly string[], diff --git a/src/core/schemas/document.test.ts b/src/core/schemas/document.test.ts index b270b62..6111b40 100644 --- a/src/core/schemas/document.test.ts +++ b/src/core/schemas/document.test.ts @@ -60,6 +60,7 @@ describe("MemoryVersionSchema", () => { provenance: "stated", source_class: "native", temporal_class: "event", + retention_class: "standard", }; const out = MemoryVersionSchema(fixture); expect(out instanceof type.errors ? out.summary : out).toEqual(fixture); diff --git a/src/core/schemas/document.ts b/src/core/schemas/document.ts index 9cd9657..5be9c89 100644 --- a/src/core/schemas/document.ts +++ b/src/core/schemas/document.ts @@ -2,6 +2,7 @@ import { type } from "arktype"; import { LINEAGE_CLASSES, PROVENANCE_MODES, + RETENTION_CLASSES, TEMPORAL_CLASSES, arktypeStringUnion, } from "../enums.ts"; @@ -30,6 +31,12 @@ export const TemporalClassSchema = type( ); export type TemporalClass = typeof TemporalClassSchema.infer; +export const RetentionClassSchema = type( + arktypeStringUnion(RETENTION_CLASSES) as + "'durable'|'standard'|'ephemeral'|'source_only'", +); +export type RetentionClass = typeof RetentionClassSchema.infer; + // The stable logical row for a captured source, deduped on (tenant_id, // adapter, external_ref). Document access is grant tags only. export const MemoryDocumentSchema = type({ @@ -70,6 +77,7 @@ export const MemoryVersionSchema = type({ provenance: ProvenanceModeSchema, source_class: LineageClassSchema, temporal_class: TemporalClassSchema, + retention_class: RetentionClassSchema, "valid_from?": "string | null", "valid_until?": "string | null", }); diff --git a/src/core/schemas/search.ts b/src/core/schemas/search.ts index 22c5c7e..dc19ee7 100644 --- a/src/core/schemas/search.ts +++ b/src/core/schemas/search.ts @@ -1,5 +1,11 @@ import { type } from "arktype"; -import { CreatedByKindSchema, MemoryVersionStatusSchema } from "./document.ts"; +import { + CreatedByKindSchema, + LineageClassSchema, + MemoryVersionStatusSchema, + ProvenanceModeSchema, + TemporalClassSchema, +} from "./document.ts"; // The retrieval contract locked on day one. A SearchHit always pins a // version_id (a citation must be reproducible against the exact version it @@ -41,6 +47,15 @@ export const SearchHitSchema = type({ citation: SearchHitCitationSchema, entity_ids: "string[]", channels_matched: SearchChannelSchema.array(), + // Additive attribution (CL-5870) — optional so older fixtures still parse. + "provenance?": ProvenanceModeSchema, + "source_class?": LineageClassSchema, + "temporal_class?": TemporalClassSchema, + "occurred_at?": "string", + "valid_until?": "string | null", + "supports?": "number.integer >= 0", + "contradicts?": "number.integer >= 0", + "derived_from?": "string[]", }); export type SearchHit = typeof SearchHitSchema.infer; diff --git a/src/db/schema.ts b/src/db/schema.ts index 9188606..2e25bba 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -80,6 +80,8 @@ export const memoryVersion = memorySchema.table( temporalClass: text("temporal_class").notNull().default("event"), validFrom: timestamp("valid_from"), validUntil: timestamp("valid_until"), + // Retention class (CL-5871) — when to forget, not how to rank. + retentionClass: text("retention_class").notNull().default("standard"), rawCaptureId: text("raw_capture_id").references(() => rawCapture.id), // Replay-generation tag — 'live' for the normal /capture path; a replay tags every // version it writes with its own transform_run id instead, so a replayed diff --git a/src/http-bodies.ts b/src/http-bodies.ts index 93b8e0e..6ad1291 100644 --- a/src/http-bodies.ts +++ b/src/http-bodies.ts @@ -36,6 +36,7 @@ export const SearchRequest = type({ "entity_ids?": "string[]", "sources?": "string[]", "includeEvidence?": "boolean", + "includeDeprecated?": "boolean", }); export type SearchRequest = typeof SearchRequest.infer; diff --git a/src/index.ts b/src/index.ts index 1312ba6..994517e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -126,6 +126,16 @@ export { type FeedResult, } from "./services/feed.ts"; +// Retention / forgetting (CL-5871) +export { + deprecateVersion, + hardDeleteDocument, + setRetentionClass, + sweepEphemeral, + tombstoneDocument, + type RetentionMutationResult, +} from "./services/retention.ts"; + // Corroboration / living relevancy (CL-5867) export { corroborationFactor, diff --git a/src/memory.ts b/src/memory.ts index 153e17d..431e1b7 100644 --- a/src/memory.ts +++ b/src/memory.ts @@ -39,6 +39,13 @@ import { type TransformConfigRow, type TransformRunRow, } from "./services/transform.ts"; +import { + deprecateVersion, + hardDeleteDocument, + setRetentionClass, + sweepEphemeral, + tombstoneDocument, +} from "./services/retention.ts"; import { documentTag, materializeShareGrants, @@ -138,6 +145,8 @@ export type MemorySearchParams = MemoryIdentity & { * Omit to include all mounted sources plus local. */ sources?: string[]; + /** Include deprecated versions in local retrieval (CL-5871). Default false. */ + includeDeprecated?: boolean; }; export type MemoryShare = ShareSugar; @@ -181,11 +190,16 @@ export type MemoryAddResult = { export type SearchAttribution = { versionId: string; provenance?: string; + sourceClass?: string; + temporalClass?: string; createdByKind?: string; generatorAgentId?: string | null; + occurredAt?: string; + validUntil?: string | null; evidence?: "strong" | "weak" | "none"; supports?: number; contradicts?: number; + derivedFrom?: string[]; }; export type SearchItem = { @@ -282,6 +296,32 @@ export type Memory = { tenantId: string; generation: string; }): Promise; + /** + * Retention write paths (engine store only). See docs/RETENTION.md (CL-5871). + */ + deprecateVersion?(input: { + tenantId: string; + versionId: string; + reason?: string; + }): Promise<{ versionId: string; documentId: string; status: string } | null>; + tombstoneDocument?(input: { + tenantId: string; + documentId: string; + reason?: string; + }): Promise<{ versions: number }>; + hardDeleteDocument?(input: { + tenantId: string; + documentId: string; + }): Promise<{ deleted: boolean; reason?: string }>; + sweepEphemeral?(input: { + tenantId: string; + now?: Date; + }): Promise<{ documentsDeleted: number }>; + setRetentionClass?(input: { + tenantId: string; + versionId: string; + retentionClass: "durable" | "standard" | "ephemeral" | "source_only"; + }): Promise<{ versionId: string; documentId: string; status: string } | null>; }; export type { TimelineEvent }; @@ -363,7 +403,10 @@ function resolveListLimit(limit: number | undefined): number | undefined { return limit; } -function hitsToSearchItems(hits: readonly SearchHit[]): SearchItem[] { +function hitsToSearchItems( + hits: readonly SearchHit[], + evidence?: HybridSearchResult["evidence"], +): SearchItem[] { return hits.map((h) => ({ documentId: h.document_id, title: h.title, @@ -377,6 +420,17 @@ function hitsToSearchItems(hits: readonly SearchHit[]): SearchItem[] { ...(h.generator_agent_id !== undefined ? { generatorAgentId: h.generator_agent_id } : {}), + ...(h.provenance !== undefined ? { provenance: h.provenance } : {}), + ...(h.source_class !== undefined ? { sourceClass: h.source_class } : {}), + ...(h.temporal_class !== undefined + ? { temporalClass: h.temporal_class } + : {}), + ...(h.occurred_at !== undefined ? { occurredAt: h.occurred_at } : {}), + ...(h.valid_until !== undefined ? { validUntil: h.valid_until } : {}), + ...(h.supports !== undefined ? { supports: h.supports } : {}), + ...(h.contradicts !== undefined ? { contradicts: h.contradicts } : {}), + ...(h.derived_from !== undefined ? { derivedFrom: h.derived_from } : {}), + ...(evidence !== undefined ? { evidence } : {}), }, })); } @@ -570,14 +624,22 @@ function mergeToSearchResult(params: { ...(params.sources !== undefined ? { sources: params.sources } : {}), }); - const items: SearchItem[] = merged.items.map((it) => ({ - documentId: it.documentId, - title: it.title, - snippet: it.snippet, - score: it.score, - kind: it.kind, - citation: it.citation, - })); + const items: SearchItem[] = merged.items.map((it) => { + // Preserve local attribution when merge kept a local hit (same documentId). + const local = params.localItems.find((l) => l.documentId === it.documentId); + return { + documentId: it.documentId, + title: it.title, + snippet: it.snippet, + score: it.score, + kind: it.kind, + citation: it.citation, + ...(local?.attribution !== undefined + ? { attribution: local.attribution } + : {}), + ...(local?.updatedAt !== undefined ? { updatedAt: local.updatedAt } : {}), + }; + }); const degraded: DegradeFlag[] = [ ...(params.localDegraded ?? []), @@ -646,6 +708,9 @@ function createPlaneFromStore( ...(params.entityIds !== undefined ? { entityIds: params.entityIds } : {}), + ...(params.includeDeprecated !== undefined + ? { includeDeprecated: params.includeDeprecated } + : {}), ...(grants !== undefined ? { grants: grants.grantStore } : {}), ...(grants?.conditionRegistry !== undefined ? { conditionRegistry: grants.conditionRegistry } @@ -659,6 +724,9 @@ function createPlaneFromStore( kind: it.kind, citation: it.citation, ...(it.updatedAt !== undefined ? { updatedAt: it.updatedAt } : {}), + ...(it.attribution !== undefined + ? { attribution: it.attribution } + : {}), })); localDegraded = local.degraded as DegradeFlag[] | undefined; localEvidence = local.evidence; @@ -887,6 +955,56 @@ function createPlaneFromStore( } return demoteGeneration(transformDeps, input); }, + + async deprecateVersion(input) { + if (!transformDeps) { + throw new MemoryError( + 501, + "retention APIs require the engine DocumentStore", + ); + } + return deprecateVersion(transformDeps.db, input); + }, + + async tombstoneDocument(input) { + if (!transformDeps) { + throw new MemoryError( + 501, + "retention APIs require the engine DocumentStore", + ); + } + return tombstoneDocument(transformDeps.db, input); + }, + + async hardDeleteDocument(input) { + if (!transformDeps) { + throw new MemoryError( + 501, + "retention APIs require the engine DocumentStore", + ); + } + return hardDeleteDocument(transformDeps.db, input); + }, + + async sweepEphemeral(input) { + if (!transformDeps) { + throw new MemoryError( + 501, + "retention APIs require the engine DocumentStore", + ); + } + return sweepEphemeral(transformDeps.db, input); + }, + + async setRetentionClass(input) { + if (!transformDeps) { + throw new MemoryError( + 501, + "retention APIs require the engine DocumentStore", + ); + } + return setRetentionClass(transformDeps.db, input); + }, }; return plane; @@ -959,6 +1077,7 @@ function createEngineDocumentStore(config: MemoryConfig): { k?: number; kinds?: string[]; entityIds?: string[]; + includeDeprecated?: boolean; grants?: DocumentStoreSearchParams["grants"]; conditionRegistry?: DocumentStoreSearchParams["conditionRegistry"]; }): Promise { @@ -973,6 +1092,9 @@ function createEngineDocumentStore(config: MemoryConfig): { ...(params.entityIds !== undefined ? { entityIds: params.entityIds } : {}), + ...(params.includeDeprecated !== undefined + ? { includeDeprecated: params.includeDeprecated } + : {}), }); if (result.hits.length === 0) return result; @@ -1112,12 +1234,15 @@ function createEngineDocumentStore(config: MemoryConfig): { ...(params.entityIds !== undefined ? { entityIds: params.entityIds } : {}), + ...(params.includeDeprecated !== undefined + ? { includeDeprecated: params.includeDeprecated } + : {}), ...(params.grants !== undefined ? { grants: params.grants } : {}), ...(params.conditionRegistry !== undefined ? { conditionRegistry: params.conditionRegistry } : {}), }); - const items = hitsToSearchItems(result.hits); + const items = hitsToSearchItems(result.hits, result.evidence); if (params.includeEvidence) { return { items, diff --git a/src/ports/types.ts b/src/ports/types.ts index 5820d28..399330c 100644 --- a/src/ports/types.ts +++ b/src/ports/types.ts @@ -47,6 +47,8 @@ export type DocumentStoreSearchParams = { kinds?: string[]; /** Narrow local retrieval by linked entity ids (unset/`[]` = no filter). */ entityIds?: string[]; + /** Include deprecated versions (CL-5871). */ + includeDeprecated?: boolean; /** * Host grant store for grant-tag document access (default engine + fakes). * Vendor stores may ignore (principal-bucket only). @@ -66,6 +68,24 @@ export type DocumentStoreSearchItem = { adapter?: string; externalRef?: string; updatedAt?: string; + /** + * Optional attribution block (CL-5870). Engine store always fills this; + * vendor stores may omit. + */ + attribution?: { + versionId: string; + provenance?: string; + sourceClass?: string; + temporalClass?: string; + createdByKind?: string; + generatorAgentId?: string | null; + occurredAt?: string; + validUntil?: string | null; + evidence?: "strong" | "weak" | "none"; + supports?: number; + contradicts?: number; + derivedFrom?: string[]; + }; }; export type DocumentStoreSearchResult = { diff --git a/src/routes/search.ts b/src/routes/search.ts index 2dc0481..68edab0 100644 --- a/src/routes/search.ts +++ b/src/routes/search.ts @@ -25,6 +25,8 @@ const SearchResponse = type({ score: "number", kind: "string", citation: "unknown", + "attribution?": "unknown", + "updatedAt?": "string", }).array(), "evidence?": "'strong'|'weak'|'none'", "degraded?": "string[]", @@ -59,8 +61,15 @@ export function mountSearchRoute(app: Hono, deps: RouteDeps): void { grantGuard(deps, "search"), validator("json", SearchRequest), async (c) => { - const { query, limit, kinds, entity_ids, sources, includeEvidence } = - c.req.valid("json"); + const { + query, + limit, + kinds, + entity_ids, + sources, + includeEvidence, + includeDeprecated, + } = c.req.valid("json"); const { scopeId, subjectId } = caller(c); try { const result = await deps.memory.search({ @@ -74,6 +83,9 @@ export function mountSearchRoute(app: Hono, deps: RouteDeps): void { ...(kinds !== undefined ? { kinds } : {}), ...(entity_ids !== undefined ? { entityIds: entity_ids } : {}), ...(sources !== undefined ? { sources } : {}), + ...(includeDeprecated !== undefined + ? { includeDeprecated } + : {}), }); return c.json(result); } catch (err) { diff --git a/src/services/retention.test.ts b/src/services/retention.test.ts new file mode 100644 index 0000000..8d73921 --- /dev/null +++ b/src/services/retention.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "bun:test"; + +// Pure policy helpers exercised via types + export surface. +// DB mutation tests need a real postgres; unit coverage is the lockstep +// enum + this smoke import. + +import { + deprecateVersion, + hardDeleteDocument, + setRetentionClass, + sweepEphemeral, + tombstoneDocument, +} from "./retention.ts"; + +describe("retention exports", () => { + it("exposes the CL-5871 write verbs", () => { + expect(typeof deprecateVersion).toBe("function"); + expect(typeof tombstoneDocument).toBe("function"); + expect(typeof hardDeleteDocument).toBe("function"); + expect(typeof sweepEphemeral).toBe("function"); + expect(typeof setRetentionClass).toBe("function"); + }); +}); diff --git a/src/services/retention.ts b/src/services/retention.ts new file mode 100644 index 0000000..80824b0 --- /dev/null +++ b/src/services/retention.ts @@ -0,0 +1,209 @@ +/** + * Retention write paths (CL-5871). + * See docs/RETENTION.md. + */ +import { and, eq, inArray, isNotNull, lt, or, sql } from "drizzle-orm"; + +import type { Db } from "../db/client.ts"; +import { memoryChunk, memoryDocument, memoryVersion } from "../db/schema.ts"; +import type { RetentionClass } from "../core/enums.ts"; + +export type RetentionMutationResult = { + versionId: string; + documentId: string; + status: string; +}; + +export async function deprecateVersion( + db: Db, + input: { + tenantId: string; + versionId: string; + reason?: string; + }, +): Promise { + const now = new Date(); + const updated = await db + .update(memoryVersion) + .set({ + status: "deprecated", + deprecatedAt: now, + deprecatedReason: input.reason ?? "deprecated", + }) + .where( + and( + eq(memoryVersion.tenantId, input.tenantId), + eq(memoryVersion.id, input.versionId), + inArray(memoryVersion.status, ["active", "superseded"]), + ), + ) + .returning({ + versionId: memoryVersion.id, + documentId: memoryVersion.documentId, + status: memoryVersion.status, + }); + return updated[0] ?? null; +} + +/** + * Tombstone: hide from search/feed, redact chunk text, keep row for audit. + * Applies to the document's live active (or deprecated) versions. + */ +export async function tombstoneDocument( + db: Db, + input: { + tenantId: string; + documentId: string; + reason?: string; + }, +): Promise<{ versions: number }> { + const now = new Date(); + const versions = await db + .update(memoryVersion) + .set({ + status: "tombstoned", + deprecatedAt: now, + deprecatedReason: input.reason ?? "tombstoned", + }) + .where( + and( + eq(memoryVersion.tenantId, input.tenantId), + eq(memoryVersion.documentId, input.documentId), + inArray(memoryVersion.status, ["active", "deprecated", "superseded"]), + ), + ) + .returning({ id: memoryVersion.id }); + + if (versions.length > 0) { + await db + .update(memoryChunk) + .set({ text: "[redacted]" }) + .where( + and( + eq(memoryChunk.tenantId, input.tenantId), + eq(memoryChunk.documentId, input.documentId), + ), + ); + } + return { versions: versions.length }; +} + +/** + * Hard-delete a document (cascade chunks/versions/edges via FKs where set). + * Blocked for durable retention_class on any non-tombstoned version. + */ +export async function hardDeleteDocument( + db: Db, + input: { + tenantId: string; + documentId: string; + }, +): Promise<{ deleted: boolean; reason?: string }> { + const durable = await db + .select({ id: memoryVersion.id }) + .from(memoryVersion) + .where( + and( + eq(memoryVersion.tenantId, input.tenantId), + eq(memoryVersion.documentId, input.documentId), + eq(memoryVersion.retentionClass, "durable"), + sql`${memoryVersion.status} <> 'tombstoned'`, + ), + ) + .limit(1); + + if (durable.length > 0) { + return { + deleted: false, + reason: "document has durable retention_class versions; tombstone first", + }; + } + + const deleted = await db + .delete(memoryDocument) + .where( + and( + eq(memoryDocument.tenantId, input.tenantId), + eq(memoryDocument.id, input.documentId), + ), + ) + .returning({ id: memoryDocument.id }); + + return { deleted: deleted.length > 0 }; +} + +/** + * Sweep ephemeral versions past valid_until (or 7d default from ingested_at). + * Hard-deletes those documents when all live versions are expired ephemeral. + */ +export async function sweepEphemeral( + db: Db, + input: { + tenantId: string; + now?: Date; + }, +): Promise<{ documentsDeleted: number }> { + const now = input.now ?? new Date(); + const expired = await db + .select({ + documentId: memoryVersion.documentId, + versionId: memoryVersion.id, + }) + .from(memoryVersion) + .where( + and( + eq(memoryVersion.tenantId, input.tenantId), + eq(memoryVersion.retentionClass, "ephemeral"), + inArray(memoryVersion.status, ["active", "deprecated", "superseded"]), + or( + and( + isNotNull(memoryVersion.validUntil), + lt(memoryVersion.validUntil, now), + ), + and( + sql`${memoryVersion.validUntil} IS NULL`, + lt( + memoryVersion.ingestedAt, + new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000), + ), + ), + ), + ), + ); + + const docIds = [...new Set(expired.map((e) => e.documentId))]; + let documentsDeleted = 0; + for (const documentId of docIds) { + const result = await hardDeleteDocument(db, { + tenantId: input.tenantId, + documentId, + }); + if (result.deleted) documentsDeleted += 1; + } + return { documentsDeleted }; +} + +export async function setRetentionClass( + db: Db, + input: { + tenantId: string; + versionId: string; + retentionClass: RetentionClass; + }, +): Promise { + const updated = await db + .update(memoryVersion) + .set({ retentionClass: input.retentionClass }) + .where( + and( + eq(memoryVersion.tenantId, input.tenantId), + eq(memoryVersion.id, input.versionId), + ), + ) + .returning({ + versionId: memoryVersion.id, + documentId: memoryVersion.documentId, + status: memoryVersion.status, + }); + return updated[0] ?? null; +} diff --git a/src/services/search.ts b/src/services/search.ts index 65cb58b..b7f93b3 100644 --- a/src/services/search.ts +++ b/src/services/search.ts @@ -142,11 +142,15 @@ export interface CandidateRow { // when a row predates the temporal migration (should not happen after 0004). temporalClass: "event" | "deadline" | "state" | "lesson"; validUntil: Date | null; + /** Lineage class (native | imported | derived) for attribution. */ + sourceClass?: string; /** Provenance mode for evidence gating (stated | inferred | unknown). */ provenance?: string; /** Independent supports / contradicts targeting this version (search-time). */ supports?: number; contradicts?: number; + /** Source version ids via derived_from edges (CL-5870). */ + derivedFrom?: string[]; } export function snippet(text: string, maxLen = 240): string { @@ -194,7 +198,7 @@ export function toHit( // authority handling baked in (the reranked path's boosted score). authorityWeight: number = AUTHORITY_WEIGHT, ): SearchHit { - return { + const hit: SearchHit = { chunk_id: row.chunkId, document_id: row.documentId, version: row.version, @@ -214,9 +218,6 @@ export function toHit( snippet: snippet(row.snippetText), kind: row.kind, created_by_kind: row.createdByKind, - ...(row.generatorAgentId - ? { generator_agent_id: row.generatorAgentId } - : {}), citation: { adapter: row.adapter, external_ref: row.externalRef, @@ -224,7 +225,25 @@ export function toHit( }, entity_ids: [], channels_matched: channelsMatched, + temporal_class: row.temporalClass, + occurred_at: row.occurredAt.toISOString(), + valid_until: row.validUntil ? row.validUntil.toISOString() : null, + supports: row.supports ?? 0, + contradicts: row.contradicts ?? 0, }; + if (row.generatorAgentId) { + hit.generator_agent_id = row.generatorAgentId; + } + if (row.provenance !== undefined) { + hit.provenance = row.provenance as NonNullable; + } + if (row.sourceClass !== undefined) { + hit.source_class = row.sourceClass as NonNullable; + } + if (row.derivedFrom !== undefined) { + hit.derived_from = row.derivedFrom; + } + return hit; } // Evidence is capped by the strong-evidence gate (authority + corroboration / @@ -372,16 +391,16 @@ export async function attachCorroborationCounts( const edges = await db .select({ - toRef: knowledgeEdge.toRef, - rel: knowledgeEdge.rel, + toRef: memoryEdge.toRef, + rel: memoryEdge.rel, }) - .from(knowledgeEdge) + .from(memoryEdge) .where( and( - eq(knowledgeEdge.tenantId, tenantId), - eq(knowledgeEdge.toType, "version"), - inArray(knowledgeEdge.toRef, versionIds), - inArray(knowledgeEdge.rel, ["supports", "contradicts"]), + eq(memoryEdge.tenantId, tenantId), + eq(memoryEdge.toType, "version"), + inArray(memoryEdge.toRef, versionIds), + inArray(memoryEdge.rel, ["supports", "contradicts"]), ), ); @@ -402,6 +421,45 @@ export async function attachCorroborationCounts( } } +/** + * Load derived_from edges (from hit version → source version) for attribution. + * Does not grant-filter sources; the DocumentStore / host may strip inaccessible + * source ids after the security post-filter if needed. + */ +export async function attachDerivedFrom( + db: Db, + tenantId: string, + rows: CandidateRow[], +): Promise { + if (rows.length === 0) return; + const versionIds = [...new Set(rows.map((r) => r.versionId))]; + const edges = await db + .select({ + fromRef: memoryEdge.fromRef, + toRef: memoryEdge.toRef, + }) + .from(memoryEdge) + .where( + and( + eq(memoryEdge.tenantId, tenantId), + eq(memoryEdge.fromType, "version"), + eq(memoryEdge.toType, "version"), + eq(memoryEdge.rel, "derived_from"), + inArray(memoryEdge.fromRef, versionIds), + ), + ); + + const map = new Map(); + for (const e of edges) { + const list = map.get(e.fromRef) ?? []; + list.push(e.toRef); + map.set(e.fromRef, list); + } + for (const row of rows) { + row.derivedFrom = map.get(row.versionId) ?? []; + } +} + // The kinds/entityIds shape shared by both channels' candidate-query params // and by HybridSearchArgs (which fans a single caller-supplied pair of these // out to both channels before fusion). An empty array is treated identically @@ -422,6 +480,8 @@ interface LexicalCandidateParams extends ChannelFilterFields { // a replayed generation's chunks never leak into a live search and vice // versa (the replay pipeline). generation?: string | undefined; + /** When true, include deprecated versions alongside active (CL-5871). */ + includeDeprecated?: boolean | undefined; } // The single FTS-candidate query for the lexical channel. Returns raw, @@ -440,11 +500,14 @@ export async function fetchLexicalCandidates( kinds, entityIds, generation = LIVE_GENERATION, + includeDeprecated = false, } = params; const conditions = [ eq(memoryChunk.tenantId, tenantId), - eq(memoryVersion.status, "active"), + includeDeprecated + ? inArray(memoryVersion.status, ["active", "deprecated"]) + : eq(memoryVersion.status, "active"), eq(memoryVersion.generation, generation), ]; @@ -498,6 +561,7 @@ export async function fetchLexicalCandidates( temporalClass: memoryVersion.temporalClass, validUntil: memoryVersion.validUntil, provenance: memoryVersion.provenance, + sourceClass: memoryVersion.sourceClass, }) .from(memoryChunk) .innerJoin( @@ -529,6 +593,7 @@ interface FetchDenseCandidatesArgs extends ChannelFilterFields { // the embed client config passed in (from that run's transform_config), // which may be only `ready` and must not require status='active'. generation?: string | undefined; + includeDeprecated?: boolean | undefined; } // Whether this pool's pgvector understands hnsw.iterative_scan, learned @@ -568,6 +633,7 @@ export async function fetchDenseCandidates( kinds, entityIds, generation = LIVE_GENERATION, + includeDeprecated = false, } = args; if (query === "") return null; @@ -637,12 +703,13 @@ export async function fetchDenseCandidates( kv.created_by_kind AS created_by_kind, kv.generator_agent_id AS generator_agent_id, c.text AS snippet_text, kv.occurred_at AS occurred_at, kv.authority AS authority, kv.temporal_class AS temporal_class, kv.valid_until AS valid_until, - kv.provenance AS provenance + kv.provenance AS provenance, kv.source_class AS source_class FROM ${activeTable.tableName} e JOIN "memory"."chunk" c ON c.id = e.chunk_id JOIN "memory"."version" kv ON kv.id = c.version_id JOIN "memory"."document" kd ON kd.id = c.document_id - WHERE e.tenant_id = $1 AND c.tenant_id = $1 AND kv.status = 'active' + WHERE e.tenant_id = $1 AND c.tenant_id = $1 + AND kv.status ${includeDeprecated ? "IN ('active', 'deprecated')" : "= 'active'"} AND kv.generation = ${generationParam} ${kindClause} ${entityClause} @@ -680,31 +747,42 @@ export async function fetchDenseCandidates( return tx.unsafe(sqlText, params as never[]); }); - return (rows as unknown as Array>).map((row) => ({ - chunkId: row["chunk_id"] as string, - documentId: row["document_id"] as string, - versionId: row["version_id"] as string, - version: row["version"] as number, - status: row["status"] as CandidateRow["status"], - title: row["title"] as string, - kind: row["kind"] as string, - adapter: row["adapter"] as string, - externalRef: row["external_ref"] as string, - createdByKind: row["created_by_kind"] as CandidateRow["createdByKind"], - generatorAgentId: (row["generator_agent_id"] as string | null) ?? null, - snippetText: row["snippet_text"] as string, - // Dense candidates carry no ts_rank-comparable score; `rank` is - // overwritten with the fused RRF score once fusion runs, and is never - // read before that. - rank: 0, - occurredAt: new Date(row["occurred_at"] as string), - authority: row["authority"] as number, - temporalClass: (row["temporal_class"] as CandidateRow["temporalClass"]) ?? "event", - validUntil: row["valid_until"] - ? new Date(row["valid_until"] as string) - : null, - provenance: (row["provenance"] as string | undefined) ?? "unknown", - })); + return (rows as unknown as Array>).map((row) => { + const sourceClass = row["source_class"] as string | null | undefined; + const provenance = row["provenance"] as string | null | undefined; + const base: CandidateRow = { + chunkId: row["chunk_id"] as string, + documentId: row["document_id"] as string, + versionId: row["version_id"] as string, + version: row["version"] as number, + status: row["status"] as CandidateRow["status"], + title: row["title"] as string, + kind: row["kind"] as string, + adapter: row["adapter"] as string, + externalRef: row["external_ref"] as string, + createdByKind: row["created_by_kind"] as CandidateRow["createdByKind"], + generatorAgentId: (row["generator_agent_id"] as string | null) ?? null, + snippetText: row["snippet_text"] as string, + // Dense candidates carry no ts_rank-comparable score; `rank` is + // overwritten with the fused RRF score once fusion runs, and is never + // read before that. + rank: 0, + occurredAt: new Date(row["occurred_at"] as string), + authority: row["authority"] as number, + temporalClass: + (row["temporal_class"] as CandidateRow["temporalClass"]) ?? "event", + validUntil: row["valid_until"] + ? new Date(row["valid_until"] as string) + : null, + }; + if (provenance != null && provenance !== "") { + base.provenance = provenance; + } + if (sourceClass != null && sourceClass !== "") { + base.sourceClass = sourceClass; + } + return base; + }); } // Vectors for the MMR diversity pass, pulled from the generation-scoped @@ -836,6 +914,8 @@ export interface HybridSearchArgs extends ChannelFilterFields { // instead, applying its transform_config's retrieval tuning when // resolvable (see resolveGenerationSearchParams, transform.ts). generation?: string | undefined; + /** Include deprecated versions in retrieval (default false). CL-5871. */ + includeDeprecated?: boolean | undefined; } const MS_PER_DAY = 24 * 60 * 60 * 1000; @@ -871,6 +951,7 @@ export async function hybridSearch( const { db, sql: rawSql, config, fetchImpl = fetch, now = new Date() } = deps; const { tenantId, principalId, kinds, entityIds } = args; const generation = args.generation ?? LIVE_GENERATION; + const includeDeprecated = args.includeDeprecated === true; const query = args.query.trim(); const k = Math.min( Math.max(1, Math.floor(args.k ?? DEFAULT_HYBRID_TOP_K)), @@ -922,6 +1003,7 @@ export async function hybridSearch( kinds, entityIds, generation, + includeDeprecated, }); const embedClientConfig = @@ -943,6 +1025,7 @@ export async function hybridSearch( kinds, entityIds, generation, + includeDeprecated, }); if (dense === null) { degraded = ["dense_unavailable"]; @@ -990,6 +1073,7 @@ export async function hybridSearch( await attachCorroborationCounts(db, tenantId, mergedRows); // Lexical-only evidence path also needs counts on the original channel rows. await attachCorroborationCounts(db, tenantId, lexicalRows); + await attachDerivedFrom(db, tenantId, mergedRows); let truncated: CandidateRow[]; // The final score per chunk, when the reranked path ran; absent on the diff --git a/src/tools/search.ts b/src/tools/search.ts index 77dd9b8..ebcd487 100644 --- a/src/tools/search.ts +++ b/src/tools/search.ts @@ -21,6 +21,9 @@ function parseSearchArgs(args: Record): MemorySearchBody { if (parsed.includeEvidence !== undefined) { body.includeEvidence = parsed.includeEvidence; } + if (parsed.includeDeprecated !== undefined) { + body.includeDeprecated = parsed.includeDeprecated; + } return body; } @@ -70,6 +73,11 @@ export const memorySearch = defineMemoryHttpTool({ description: "Include evidence strength on the response (hub default true)", }, + includeDeprecated: { + type: "boolean", + description: + "Include deprecated versions in results (default false)", + }, }, required: ["query"], additionalProperties: false, From 2bef6bb7674dea4c4866e7a760e947f25ad715e3 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 10 Aug 2026 10:50:14 -0700 Subject: [PATCH 10/19] fix: TTL sweep deprecates ephemeral versions (does not hard-delete) Matches CL-5871 plan: host-schedulable sweep auto-deprecates; hard-delete stays an explicit verb. --- docs/RETENTION.md | 5 +++-- src/memory.ts | 2 +- src/services/retention.ts | 30 ++++++++++++------------------ 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/docs/RETENTION.md b/docs/RETENTION.md index 6d3d86e..fe230cd 100644 --- a/docs/RETENTION.md +++ b/docs/RETENTION.md @@ -21,11 +21,12 @@ CHECK constraint `version_retention_class_check` stays lockstep with | Deprecate | `memory.deprecateVersion` | `status=deprecated`, `deprecated_at` / reason | | Tombstone | `memory.tombstoneDocument` | All active/deprecated/superseded versions → `tombstoned`; chunk text redacted to `[redacted]` | | Hard delete | `memory.hardDeleteDocument` | Deletes document row (cascade); **refuses** if any non-tombstoned version is `durable` | -| Sweep | `memory.sweepEphemeral` | Hard-deletes documents whose ephemeral versions are past TTL | +| Sweep | `memory.sweepEphemeral` | Auto-deprecates ephemeral versions past `valid_until` (or 7d from `ingested_at`); host schedules, core is cron-free | | Set class | `memory.setRetentionClass` | Update `retention_class` on a version | Search and feed exclude non-active (and non-superseded for feed) rows by default. Pass `includeDeprecated: true` on search to retrieve deprecated -versions intentionally (ops / audit). +versions intentionally (ops / audit). Hard-delete is a separate explicit +verb — TTL never hard-deletes. Service module: `src/services/retention.ts`. diff --git a/src/memory.ts b/src/memory.ts index 431e1b7..d7bc9e8 100644 --- a/src/memory.ts +++ b/src/memory.ts @@ -316,7 +316,7 @@ export type Memory = { sweepEphemeral?(input: { tenantId: string; now?: Date; - }): Promise<{ documentsDeleted: number }>; + }): Promise<{ versionsDeprecated: number }>; setRetentionClass?(input: { tenantId: string; versionId: string; diff --git a/src/services/retention.ts b/src/services/retention.ts index 80824b0..719f4d7 100644 --- a/src/services/retention.ts +++ b/src/services/retention.ts @@ -134,7 +134,8 @@ export async function hardDeleteDocument( /** * Sweep ephemeral versions past valid_until (or 7d default from ingested_at). - * Hard-deletes those documents when all live versions are expired ephemeral. + * Auto-deprecates expired ephemeral versions (core stays cron-free — host + * schedules this). Hard-delete remains an explicit separate verb. */ export async function sweepEphemeral( db: Db, @@ -142,19 +143,20 @@ export async function sweepEphemeral( tenantId: string; now?: Date; }, -): Promise<{ documentsDeleted: number }> { +): Promise<{ versionsDeprecated: number }> { const now = input.now ?? new Date(); const expired = await db - .select({ - documentId: memoryVersion.documentId, - versionId: memoryVersion.id, + .update(memoryVersion) + .set({ + status: "deprecated", + deprecatedAt: now, + deprecatedReason: "ephemeral_ttl", }) - .from(memoryVersion) .where( and( eq(memoryVersion.tenantId, input.tenantId), eq(memoryVersion.retentionClass, "ephemeral"), - inArray(memoryVersion.status, ["active", "deprecated", "superseded"]), + eq(memoryVersion.status, "active"), or( and( isNotNull(memoryVersion.validUntil), @@ -169,18 +171,10 @@ export async function sweepEphemeral( ), ), ), - ); + ) + .returning({ id: memoryVersion.id }); - const docIds = [...new Set(expired.map((e) => e.documentId))]; - let documentsDeleted = 0; - for (const documentId of docIds) { - const result = await hardDeleteDocument(db, { - tenantId: input.tenantId, - documentId, - }); - if (result.deleted) documentsDeleted += 1; - } - return { documentsDeleted }; + return { versionsDeprecated: expired.length }; } export async function setRetentionClass( From d322fa6f2c28331b524fc9812f96819cd8e2b353 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 10 Aug 2026 10:51:46 -0700 Subject: [PATCH 11/19] test: attribution wire coverage + close Level 4 residual notes Unit-test toHit/SearchHitSchema attribution shapes; document list without attribution; feed status filter comment reflects 4b retention outcomes; memory_search tool blurb mentions attribution guidance. --- IMPLEMENTATION.md | 2 + src/core/schemas/search.test.ts | 22 +++++++++++ src/services/feed.ts | 4 +- src/services/search.test.ts | 67 +++++++++++++++++++++++++++++++++ src/tools/search.ts | 6 ++- 5 files changed, 97 insertions(+), 4 deletions(-) diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md index 3cca305..5cc4b93 100644 --- a/IMPLEMENTATION.md +++ b/IMPLEMENTATION.md @@ -415,6 +415,8 @@ Capture-time `authority` is never rewritten. See `docs/RELEVANCY.md`. onto candidates; `toHit` emits provenance, source/temporal class, occurred_at, valid_until, corroboration counts, and derived_from. The plane maps these into additive `SearchItem.attribution` (and `DocumentStoreSearchItem.attribution`). +**List/timeline** stays document-title oriented and does not attach the full +attribution block (version-level fields are search/feed concerns). **Retention (CL-5871):** `memory.version.retention_class` + migration `0007_retention.sql`. Plane helpers in `services/retention.ts` diff --git a/src/core/schemas/search.test.ts b/src/core/schemas/search.test.ts index 0ce09fb..79f45d3 100644 --- a/src/core/schemas/search.test.ts +++ b/src/core/schemas/search.test.ts @@ -47,6 +47,28 @@ describe("SearchHitSchema", () => { const out = SearchHitSchema(fixture); expect(out instanceof type.errors).toBe(true); }); + + it("accepts additive attribution fields (CL-5870)", () => { + const fixture = fullHitFixture(); + fixture.provenance = "inferred"; + fixture.source_class = "derived"; + fixture.temporal_class = "lesson"; + fixture.occurred_at = "2026-07-01T00:00:00.000Z"; + fixture.valid_until = null; + fixture.supports = 3; + fixture.contradicts = 1; + fixture.derived_from = ["kv_src"]; + fixture.generator_agent_id = "resident-distiller"; + fixture.created_by_kind = "agent"; + const out = SearchHitSchema(fixture); + expect(out instanceof type.errors ? out.summary : out).toEqual(fixture); + }); + + it("still parses without attribution fields (additive / back-compat)", () => { + const fixture = fullHitFixture(); + const out = SearchHitSchema(fixture); + expect(out instanceof type.errors).toBe(false); + }); }); describe("SearchResponseSchema", () => { diff --git a/src/services/feed.ts b/src/services/feed.ts index 4e0650b..f56347e 100644 --- a/src/services/feed.ts +++ b/src/services/feed.ts @@ -69,8 +69,8 @@ export async function fetchFeed( const conditions = [ eq(memoryVersion.tenantId, args.tenantId), eq(memoryVersion.generation, LIVE_GENERATION), - // Active (or superseded historical) live rows — exclude tombstones later in 4b. - // Feed surfaces commits; status filter keeps deprecated/tombstoned out once written. + // Live feed: active or superseded commits only. Deprecated / tombstoned + // versions are retention write-path outcomes and stay out of the cursor stream. sql`${memoryVersion.status} IN ('active', 'superseded')`, gt(memoryVersion.feedSeq, after), ]; diff --git a/src/services/search.test.ts b/src/services/search.test.ts index af81e5f..0fa6d87 100644 --- a/src/services/search.test.ts +++ b/src/services/search.test.ts @@ -6,6 +6,7 @@ import { fetchDenseCandidates, hnswEfSearch, snippet, + toHit, type CandidateRow, } from "./search.ts"; @@ -522,3 +523,69 @@ describe("fetchDenseCandidates kind/entity filtering", () => { expect(chunkIds).toContain("chunk-note"); }); }); + +describe("toHit — wire attribution (CL-5870)", () => { + it("surfaces provenance, temporal, corroboration, and derived_from on the hit", () => { + const hit = toHit( + candidate({ + provenance: "inferred", + sourceClass: "derived", + temporalClass: "state", + validUntil: new Date("2026-12-01T00:00:00Z"), + supports: 2, + contradicts: 0, + derivedFrom: ["kv_source_1", "kv_source_2"], + generatorAgentId: "resident-distiller", + createdByKind: "agent", + }), + ); + expect(hit.version_id).toBe("ver_1"); + expect(hit.provenance).toBe("inferred"); + expect(hit.source_class).toBe("derived"); + expect(hit.temporal_class).toBe("state"); + expect(hit.valid_until).toBe("2026-12-01T00:00:00.000Z"); + expect(hit.supports).toBe(2); + expect(hit.contradicts).toBe(0); + expect(hit.derived_from).toEqual(["kv_source_1", "kv_source_2"]); + expect(hit.generator_agent_id).toBe("resident-distiller"); + expect(hit.created_by_kind).toBe("agent"); + }); + + it("omits optional attribution fields when absent (additive wire)", () => { + const hit = toHit(candidate()); + expect(hit.provenance).toBeUndefined(); + expect(hit.source_class).toBeUndefined(); + expect(hit.derived_from).toBeUndefined(); + expect(hit.generator_agent_id).toBeUndefined(); + expect(hit.supports).toBe(0); + expect(hit.contradicts).toBe(0); + expect(hit.temporal_class).toBe("event"); + }); + + it("distinguishes stated human vs inferred agent attribution shapes", () => { + const human = toHit( + candidate({ + provenance: "stated", + sourceClass: "native", + createdByKind: "human", + generatorAgentId: null, + }), + ); + const distilled = toHit( + candidate({ + provenance: "inferred", + sourceClass: "derived", + createdByKind: "agent", + generatorAgentId: "resident-distiller", + derivedFrom: ["kv_raw"], + }), + ); + expect(human.provenance).toBe("stated"); + expect(human.created_by_kind).toBe("human"); + expect(human.generator_agent_id).toBeUndefined(); + expect(distilled.provenance).toBe("inferred"); + expect(distilled.created_by_kind).toBe("agent"); + expect(distilled.generator_agent_id).toBe("resident-distiller"); + expect(distilled.derived_from).toEqual(["kv_raw"]); + }); +}); diff --git a/src/tools/search.ts b/src/tools/search.ts index ebcd487..6814517 100644 --- a/src/tools/search.ts +++ b/src/tools/search.ts @@ -37,8 +37,10 @@ export const memorySearch = defineMemoryHttpTool({ name: "memory_search", description: "Hybrid semantic + keyword search over tenant memory. " + - "Returns ranked items (and optional evidence). Identity is " + - "the authenticated principal on the hub.", + "Returns ranked items with optional additive attribution " + + "(provenance, temporal class, corroboration, derivedFrom) and evidence. " + + "Attribute stated content to the actor; treat inferred as own-voice claims. " + + "Identity is the authenticated principal on the hub.", inputSchema: { type: "object", properties: { From 6dc66f2aec3762faa565c0f1a4405b0403f41cbd Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 10 Aug 2026 11:08:51 -0700 Subject: [PATCH 12/19] feat: first-class resident distiller export (CL-5869) Ship @corbits/memory/distiller so Corbits apps can opt into continuous distillation with createResidentDistiller or runDistillTick. Claim-aware add + memory_feed tool + accessTags on feed entries complete the substrate. Inference stays host-injected. --- CHANGELOG.md | 6 +- PRODUCT.md | 13 ++- bun.lock | 1 + docs/DISTILLER.md | 110 ++++++++++++++----- package.json | 6 +- src/distiller/claim.test.ts | 56 ++++++++++ src/distiller/claim.ts | 89 ++++++++++++++++ src/distiller/constants.ts | 8 ++ src/distiller/index.ts | 43 ++++++++ src/distiller/tick.test.ts | 142 +++++++++++++++++++++++++ src/distiller/tick.ts | 187 +++++++++++++++++++++++++++++++++ src/distiller/workflow.test.ts | 44 ++++++++ src/distiller/workflow.ts | 116 ++++++++++++++++++++ src/http-bodies.ts | 10 ++ src/index.ts | 28 ++++- src/memory.ts | 76 +++++++++++++- src/ports/types.ts | 10 ++ src/routes/add.ts | 22 ++++ src/routes/feed.ts | 1 + src/tools/add.ts | 73 ++++++++++++- src/tools/client.ts | 25 +++++ src/tools/feed.ts | 78 ++++++++++++++ src/tools/index.ts | 2 + 23 files changed, 1103 insertions(+), 43 deletions(-) create mode 100644 src/distiller/claim.test.ts create mode 100644 src/distiller/claim.ts create mode 100644 src/distiller/constants.ts create mode 100644 src/distiller/index.ts create mode 100644 src/distiller/tick.test.ts create mode 100644 src/distiller/tick.ts create mode 100644 src/distiller/workflow.test.ts create mode 100644 src/distiller/workflow.ts create mode 100644 src/tools/feed.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index db69796..fe46dea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,7 +76,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `deprecateVersion` / `tombstoneDocument` / `hardDeleteDocument` / `sweepEphemeral` / `setRetentionClass`; `includeDeprecated` on search. See `docs/RETENTION.md` -- Distiller substrate notes: `docs/DISTILLER.md` (CL-5869 host workflow) +- **Resident distiller (CL-5869):** `@corbits/memory/distiller` — + `createResidentDistiller({ inference })` schedule workflow + `runDistillTick` + + `buildDistilledClaim`. Claim-aware `add` (generator_agent_id, provenance, + derived_from, …). `memory_feed` tool. Feed entries include `accessTags`. + See `docs/DISTILLER.md` ### Removed diff --git a/PRODUCT.md b/PRODUCT.md index 743d388..b0a76e0 100644 --- a/PRODUCT.md +++ b/PRODUCT.md @@ -4,8 +4,9 @@ Memory for Interchange hubs: durable documents, hybrid search, recent list. **You mount it on the hub (~5 lines). That exposes protected routes. Agents and ingestion modules call those routes.** Workbench and coding agents are -clients — not owners of ingestion or auth. Inference is host-owned (call your -model, then `add` / `search`); core does not ship an answer endpoint. +clients — not owners of auth. Inference stays host-injected; the package ships +add/search/list **and** an optional resident distiller so apps can opt into +continuous distillation with a few lines (`docs/DISTILLER.md`). ## Shape (locked) @@ -18,7 +19,8 @@ never creates one; it mounts onto yours. | `loadMemoryConfig()` | Config from env | | `runMemoryMigrations(url)` | Apply pgvector schema | | `registerMemoryRoutes` | Low-level HTTP only (optional) | -| `@corbits/memory/tools` | Interchange `defineTool` factories (`memory_add` / `memory_search` / `memory_list`) | +| `@corbits/memory/tools` | Interchange tools (`memory_add` / `search` / `list` / `feed`) | +| `@corbits/memory/distiller` | `createResidentDistiller` workflow + `runDistillTick` | ### Verbs @@ -31,8 +33,9 @@ never creates one; it mounts onto yours. Engine-only plane helpers (no HTTP yet): transform/replay, retention (`deprecateVersion` / `tombstoneDocument` / … — see `docs/RETENTION.md`), -share-grant materialization. Distiller is a **host** `onTrigger` workflow -using feed + add + attribution (`docs/DISTILLER.md`). +share-grant materialization. Distiller is first-class: +`createResidentDistiller` / `runDistillTick` (`docs/DISTILLER.md`) — host +injects inference; package ships the workflow + tick helpers. Identity is always **`principalId` + `tenantId`** on the plane. HTTP routes never take body identity — they read `c.get("principal")` from Interchange diff --git a/bun.lock b/bun.lock index eeeb56d..a2bdc12 100644 --- a/bun.lock +++ b/bun.lock @@ -9,6 +9,7 @@ "@intx/authz": "0.2.2", "@intx/hub-api": "0.2.2", "@intx/log": "0.2.2", + "@intx/workflow": "0.2.2", "arktype": "^2.1.29", "drizzle-orm": "^0.45.1", "hono": "^4.9.0", diff --git a/docs/DISTILLER.md b/docs/DISTILLER.md index dcaea01..1416034 100644 --- a/docs/DISTILLER.md +++ b/docs/DISTILLER.md @@ -1,38 +1,92 @@ -# Resident distiller (CL-5869) +# Resident distiller -The **resident distiller** is a host workflow (Interchange `onTrigger`), not a -process inside this package. Memory exposes the **substrate** only: +First-class on-ramp so a Corbits app can add memory **and** keep it distilled +without a sibling package. -| Substrate | Where | +## Quick start (Interchange workflow) + +```ts +import { createResidentDistiller } from "@corbits/memory/distiller"; +// or: import { createResidentDistiller } from "@corbits/memory"; + +const { workflow, generatorAgentId } = createResidentDistiller({ + inference: { + sources: [{ provider: "openai", model: "gpt-4.1-mini" }], + }, + // optional: cron: "*/5 * * * *", id, agentId, systemPrompt, extraTools +}); + +// Deploy `workflow` with host workflow-deploy. +// Env for tools: memoryBaseUrl, memoryTenantId, memoryAuthToken +// Grant the distiller principal: memory:search + memory:add (feed via search grant) +``` + +The agent is preloaded with `memory_feed`, `memory_add`, and `memory_search`. +System prompt encodes loop-safety (`exclude_generator` = `generatorAgentId`), +access-tag copy (never widen), and fail-soft poison handling. + +## Quick start (imperative — any scheduler) + +```ts +import { runDistillTick } from "@corbits/memory/distiller"; +import { createMemoryHttpClient } from "@corbits/memory/tools"; + +const client = createMemoryHttpClient({ + baseUrl: process.env.MEMORY_BASE_URL!, + tenantId: process.env.MEMORY_TENANT_ID!, + authToken: process.env.MEMORY_AUTH_TOKEN!, +}); + +let cursor = 0; +const result = await runDistillTick({ + client, + after: cursor, + distill: async (entry) => { + // call your model — return skip | poison | write + return { + action: "write", + title: "Claim", + text: "…", + temporalClass: "lesson", + }; + }, +}); +cursor = result.nextCursor; // persist +``` + +Inference is **always injected** (`distill` callback or host agent sources). +The package never embeds a model. + +## Helpers + +| Export | Use | +| --- | --- | +| `buildDistilledClaim` | Wire body with `generator_agent_id`, `provenance=inferred`, `derived_from` | +| `shouldProcessFeedEntry` | Skip own generator writes (defense in depth) | +| `resolveNextCursor` | Fail-soft cursor advance after poison | +| `RESIDENT_DISTILLER_AGENT_ID` | Default `"resident-distiller"` | + +## Substrate (already on the plane) + +| Piece | Where | | --- | --- | -| Capture feed (exactly-once cursor) | `memory.feed` / `GET .../memory/feed` — [FEED.md](./FEED.md) | -| Claim-bearing + provenance | version columns — claim-bearing schema | -| Temporal classes | [TEMPORAL.md](./TEMPORAL.md) | -| Corroboration / living relevancy | [RELEVANCY.md](./RELEVANCY.md) | -| Wire attribution on search | `SearchItem.attribution` (CL-5870) | +| Capture feed (exactly-once cursor) | `memory.feed` / `GET …/memory/feed` — [FEED.md](./FEED.md) | +| Claim identity on add | `generator_agent_id`, `provenance`, `lineage_class`, `derived_from` | +| Wire attribution on search | `SearchItem.attribution` | | Retention / forgetting | [RETENTION.md](./RETENTION.md) | -| Transform / staged replay | `runTransform` / promote / demote | -| Share grants | `share.principals` → WritableGrantStore | +| Tools | `@corbits/memory/tools` — `memoryAdd`, `memoryFeed`, `memorySearch`, `memoryList` | -## Recommended body shape +## Grant manifest (host) -1. **Pull** — `feed({ after: cursor, excludeGenerator: "resident-distiller" })` -2. **Classify / gate** — host policy (action-authority, kinds, poison skip) -3. **Distill** — host LLM; write via `add` with agent identity and - `generatorAgentId: "resident-distiller"`, provenance `inferred` as needed -4. **Link** — supports/contradicts edges (host write path) -5. **Attribute consumers** — search hits surface provenance, temporal class, - corroboration counts, and `derivedFrom` version ids for UI/citation -6. **Advance cursor** — store `nextCursor` only after successful handling - (or after fail-soft poison quarantine) +Minimum capabilities for the distiller principal: -Loop-safety: always pass `excludeGenerator` matching the distiller’s -`generatorAgentId` so the feed never re-delivers the distiller’s own writes. +- `memory:search` (covers feed pull under the same capability family hosts use today) +- `memory:add` (claim writes) -## Out of scope here +Copy `accessTags` from each feed entry onto writes — never mint broader tags. -- Deploying the workflow, model choice, grant manifest contents beyond tags -- Push outbox (phase 2 of the feed — still pull-only) -- Automatic edge minting on add +## Out of scope -See `dispatch/resident-memory-distillation/5a-distiller_workflow/plan.md`. +- Host deploy pipeline / secrets +- Push outbox (feed remains pull-only) +- Automatic supports/contradicts edge minting beyond `derived_from` on add diff --git a/package.json b/package.json index d754bed..0065269 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,13 @@ { "name": "@corbits/memory", "version": "0.1.2", - "description": "Mountable memory add/search/list SDK for Interchange hubs", + "description": "Mountable memory add/search/list SDK for Interchange hubs — includes resident distiller", "exports": { ".": "./src/index.ts", "./migrations": "./src/migrations.ts", "./config": "./src/mount-config.ts", - "./tools": "./src/tools/index.ts" + "./tools": "./src/tools/index.ts", + "./distiller": "./src/distiller/index.ts" }, "interchange": { "tools": "./src/tools/index.ts" @@ -28,6 +29,7 @@ "@intx/authz": "0.2.2", "@intx/hub-api": "0.2.2", "@intx/log": "0.2.2", + "@intx/workflow": "0.2.2", "arktype": "^2.1.29", "drizzle-orm": "^0.45.1", "hono": "^4.9.0", diff --git a/src/distiller/claim.test.ts b/src/distiller/claim.test.ts new file mode 100644 index 0000000..d4e18be --- /dev/null +++ b/src/distiller/claim.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "bun:test"; + +import { + buildDistilledClaim, + resolveNextCursor, + shouldProcessFeedEntry, +} from "./claim.ts"; +import { RESIDENT_DISTILLER_AGENT_ID } from "./constants.ts"; + +describe("buildDistilledClaim", () => { + it("sets inferred/derived identity and copies access tags", () => { + const claim = buildDistilledClaim({ + title: "Decision", + text: "Ship the feed first", + sourceAccessTags: ["memory.owner:u1", "memory.doc:d1"], + derivedFromVersionIds: ["kv_src"], + temporalClass: "lesson", + }); + expect(claim.generator_agent_id).toBe(RESIDENT_DISTILLER_AGENT_ID); + expect(claim.provenance).toBe("inferred"); + expect(claim.lineage_class).toBe("derived"); + expect(claim.derived_from).toEqual(["kv_src"]); + expect(claim.access_tags).toEqual(["memory.owner:u1", "memory.doc:d1"]); + expect(claim.temporal_class).toBe("lesson"); + }); +}); + +describe("shouldProcessFeedEntry", () => { + it("skips own generator writes", () => { + expect( + shouldProcessFeedEntry({ + versionId: "v1", + generatorAgentId: RESIDENT_DISTILLER_AGENT_ID, + }), + ).toBe(false); + }); + + it("accepts human / other agent writes", () => { + expect( + shouldProcessFeedEntry({ versionId: "v1", generatorAgentId: null }), + ).toBe(true); + expect( + shouldProcessFeedEntry({ + versionId: "v1", + generatorAgentId: "other-bot", + }), + ).toBe(true); + }); +}); + +describe("resolveNextCursor", () => { + it("returns page cursor even on poison (fail-soft)", () => { + expect(resolveNextCursor({ nextCursor: 42 }, { poison: true })).toBe(42); + expect(resolveNextCursor({ nextCursor: null })).toBe(null); + }); +}); diff --git a/src/distiller/claim.ts b/src/distiller/claim.ts new file mode 100644 index 0000000..38c148f --- /dev/null +++ b/src/distiller/claim.ts @@ -0,0 +1,89 @@ +import { RESIDENT_DISTILLER_AGENT_ID } from "./constants.ts"; + +/** + * Wire body fragment for a distilled claim write (memory_add / HTTP add). + * Access tags must be ≤ source entry tags — never widen. + */ +export type DistilledClaimWrite = { + title: string; + text: string; + access_tags: string[]; + generator_agent_id: string; + provenance: "inferred"; + lineage_class: "derived"; + derived_from: string[]; + kind?: string; + temporal_class?: "event" | "deadline" | "state" | "lesson"; + valid_from?: string; + valid_until?: string; +}; + +export type BuildDistilledClaimArgs = { + title: string; + text: string; + /** Source feed entry access tags (copied, not widened). */ + sourceAccessTags: readonly string[]; + /** Source version id(s) this claim is derived from. */ + derivedFromVersionIds: readonly string[]; + generatorAgentId?: string; + kind?: string; + temporalClass?: "event" | "deadline" | "state" | "lesson"; + validFrom?: string; + validUntil?: string; +}; + +/** Build a claim write body with loop-safe generator id and derived lineage. */ +export function buildDistilledClaim( + args: BuildDistilledClaimArgs, +): DistilledClaimWrite { + const claim: DistilledClaimWrite = { + title: args.title, + text: args.text, + access_tags: [...args.sourceAccessTags], + generator_agent_id: args.generatorAgentId ?? RESIDENT_DISTILLER_AGENT_ID, + provenance: "inferred", + lineage_class: "derived", + derived_from: [...args.derivedFromVersionIds], + }; + if (args.kind !== undefined) claim.kind = args.kind; + if (args.temporalClass !== undefined) { + claim.temporal_class = args.temporalClass; + } + if (args.validFrom !== undefined) claim.valid_from = args.validFrom; + if (args.validUntil !== undefined) claim.valid_until = args.validUntil; + return claim; +} + +export type FeedEntryLike = { + versionId: string; + generatorAgentId?: string | null; + kind?: string; + title?: string; + accessTags?: readonly string[]; +}; + +/** + * Gate: skip own writes (defense in depth — feed excludeGenerator is primary). + * Host policy can wrap this for kind/action-authority filters. + */ +export function shouldProcessFeedEntry( + entry: FeedEntryLike, + generatorAgentId: string = RESIDENT_DISTILLER_AGENT_ID, +): boolean { + if (entry.generatorAgentId === generatorAgentId) return false; + return true; +} + +/** + * Cursor advance: after processing a page, store nextCursor only when the + * host finished handling (including fail-soft poison quarantine). + * Pure helper — no I/O. + */ +export function resolveNextCursor( + page: { nextCursor: number | null }, + opts?: { poison?: boolean }, +): number | null { + // Fail-soft: still advance so a poison entry cannot block the feed forever. + if (opts?.poison) return page.nextCursor; + return page.nextCursor; +} diff --git a/src/distiller/constants.ts b/src/distiller/constants.ts new file mode 100644 index 0000000..ba9dea0 --- /dev/null +++ b/src/distiller/constants.ts @@ -0,0 +1,8 @@ +/** Stable generator id for the resident distiller — use with feed excludeGenerator. */ +export const RESIDENT_DISTILLER_AGENT_ID = "resident-distiller"; + +/** Default schedule: every 5 minutes. */ +export const RESIDENT_DISTILLER_CRON_DEFAULT = "*/5 * * * *"; + +/** Default workflow id when host does not override. */ +export const RESIDENT_DISTILLER_WORKFLOW_ID = "resident-memory-distiller"; diff --git a/src/distiller/index.ts b/src/distiller/index.ts new file mode 100644 index 0000000..4f63e78 --- /dev/null +++ b/src/distiller/index.ts @@ -0,0 +1,43 @@ +/** + * Resident distiller — first-class DX for Corbits apps. + * + * Two ways to run: + * + * 1. **Workflow** (recommended on Interchange): + * `createResidentDistiller({ inference })` → deploy the workflow. + * + * 2. **Imperative tick** (any scheduler): + * `runDistillTick({ client, distill, after })` with your model in `distill`. + * + * Substrate (feed, claim-bearing add, attribution, retention) lives in the + * memory plane; this module is the easy on-ramp, not a separate package. + */ +export { + RESIDENT_DISTILLER_AGENT_ID, + RESIDENT_DISTILLER_CRON_DEFAULT, + RESIDENT_DISTILLER_WORKFLOW_ID, +} from "./constants.ts"; + +export { + buildDistilledClaim, + resolveNextCursor, + shouldProcessFeedEntry, + type BuildDistilledClaimArgs, + type DistilledClaimWrite, + type FeedEntryLike, +} from "./claim.ts"; + +export { + runDistillTick, + type DistillOutcome, + type DistillTickFeedEntry, + type DistillTickPage, + type DistillTickResult, + type RunDistillTickArgs, +} from "./tick.ts"; + +export { + createResidentDistiller, + type CreateResidentDistillerOpts, + type ResidentDistiller, +} from "./workflow.ts"; diff --git a/src/distiller/tick.test.ts b/src/distiller/tick.test.ts new file mode 100644 index 0000000..c85383c --- /dev/null +++ b/src/distiller/tick.test.ts @@ -0,0 +1,142 @@ +import { describe, expect, it } from "bun:test"; + +import type { MemoryHttpClient } from "../tools/client.ts"; +import { RESIDENT_DISTILLER_AGENT_ID } from "./constants.ts"; +import { runDistillTick } from "./tick.ts"; + +function fakeClient(opts: { + page: unknown; + adds?: unknown[]; +}): MemoryHttpClient { + const adds = opts.adds ?? []; + return { + async add(body) { + adds.push(body); + return { documentId: "d_new", versionId: "v_new" }; + }, + async search() { + return { items: [] }; + }, + async list() { + return { events: [] }; + }, + async feed() { + return opts.page; + }, + }; +} + +describe("runDistillTick", () => { + it("writes claims and advances cursor", async () => { + const adds: unknown[] = []; + const client = fakeClient({ + adds, + page: { + entries: [ + { + feedSeq: 1, + versionId: "v1", + documentId: "d1", + kind: "note", + title: "Raw", + status: "active", + createdByKind: "human", + generatorAgentId: null, + provenance: "stated", + occurredAt: "2026-01-01T00:00:00.000Z", + createdAt: "2026-01-01T00:00:00.000Z", + accessTags: ["memory.owner:u1"], + }, + ], + nextCursor: 1, + }, + }); + + const result = await runDistillTick({ + client, + after: 0, + distill: async () => ({ + action: "write", + title: "Claim", + text: "Durable fact", + temporalClass: "state", + }), + }); + + expect(result.nextCursor).toBe(1); + expect(result.wrote).toBe(1); + expect(result.skipped).toBe(0); + expect(adds).toHaveLength(1); + const body = adds[0] as Record; + expect(body["generator_agent_id"]).toBe(RESIDENT_DISTILLER_AGENT_ID); + expect(body["derived_from"]).toEqual(["v1"]); + expect(body["access_tags"]).toEqual(["memory.owner:u1"]); + }); + + it("fail-soft poisons and still advances", async () => { + const client = fakeClient({ + page: { + entries: [ + { + feedSeq: 3, + versionId: "v3", + documentId: "d3", + kind: "note", + title: "Bad", + status: "active", + createdByKind: "human", + generatorAgentId: null, + provenance: "stated", + occurredAt: "2026-01-01T00:00:00.000Z", + createdAt: "2026-01-01T00:00:00.000Z", + accessTags: [], + }, + ], + nextCursor: 3, + }, + }); + + const result = await runDistillTick({ + client, + after: 2, + distill: async () => { + throw new Error("model blew up"); + }, + }); + + expect(result.poisoned).toBe(1); + expect(result.wrote).toBe(0); + expect(result.nextCursor).toBe(3); + }); + + it("skips when distill returns skip", async () => { + const client = fakeClient({ + page: { + entries: [ + { + feedSeq: 2, + versionId: "v2", + documentId: "d2", + kind: "note", + title: "Noise", + status: "active", + createdByKind: "human", + generatorAgentId: null, + provenance: "stated", + occurredAt: "2026-01-01T00:00:00.000Z", + createdAt: "2026-01-01T00:00:00.000Z", + accessTags: [], + }, + ], + nextCursor: 2, + }, + }); + + const result = await runDistillTick({ + client, + distill: async () => ({ action: "skip" }), + }); + expect(result.skipped).toBe(1); + expect(result.wrote).toBe(0); + }); +}); diff --git a/src/distiller/tick.ts b/src/distiller/tick.ts new file mode 100644 index 0000000..aca4c62 --- /dev/null +++ b/src/distiller/tick.ts @@ -0,0 +1,187 @@ +/** + * Imperative distill tick — for hosts that want a function, not a workflow. + * Inference stays injected: host supplies `distill` (call your model). + */ +import type { MemoryHttpClient } from "../tools/client.ts"; +import { + buildDistilledClaim, + shouldProcessFeedEntry, + type DistilledClaimWrite, +} from "./claim.ts"; +import { RESIDENT_DISTILLER_AGENT_ID } from "./constants.ts"; + +export type DistillTickFeedEntry = { + feedSeq: number; + versionId: string; + documentId: string; + kind: string; + title: string; + status: string; + createdByKind: string; + generatorAgentId: string | null; + provenance: string; + occurredAt: string; + createdAt: string; + accessTags: string[]; +}; + +export type DistillTickPage = { + entries: DistillTickFeedEntry[]; + nextCursor: number | null; +}; + +export type DistillOutcome = + | { action: "skip" } + | { action: "poison"; reason?: string } + | { + action: "write"; + title: string; + text: string; + kind?: string; + temporalClass?: "event" | "deadline" | "state" | "lesson"; + }; + +export type RunDistillTickArgs = { + client: MemoryHttpClient; + /** Exclusive cursor (last processed feed_seq). Default 0. */ + after?: number; + limit?: number; + generatorAgentId?: string; + /** + * Host inference: classify + distill one feed entry. + * Return skip / poison / write. Never throw for poison — use { action: "poison" }. + */ + distill: (entry: DistillTickFeedEntry) => Promise; + signal?: AbortSignal; +}; + +export type DistillTickResult = { + /** Cursor to persist for the next tick. */ + nextCursor: number; + processed: number; + wrote: number; + skipped: number; + poisoned: number; + claims: DistilledClaimWrite[]; +}; + +function parseFeedPage(raw: unknown): DistillTickPage { + if (raw === null || typeof raw !== "object") { + throw new Error("distill tick: feed response is not an object"); + } + const o = raw as Record; + const entriesRaw = o["entries"]; + if (!Array.isArray(entriesRaw)) { + throw new Error("distill tick: feed.entries missing"); + } + const entries: DistillTickFeedEntry[] = entriesRaw.map((e, i) => { + if (e === null || typeof e !== "object") { + throw new Error(`distill tick: feed.entries[${i}] invalid`); + } + const row = e as Record; + return { + feedSeq: Number(row["feedSeq"]), + versionId: String(row["versionId"] ?? ""), + documentId: String(row["documentId"] ?? ""), + kind: String(row["kind"] ?? "note"), + title: String(row["title"] ?? ""), + status: String(row["status"] ?? "active"), + createdByKind: String(row["createdByKind"] ?? "system"), + generatorAgentId: + row["generatorAgentId"] === null || row["generatorAgentId"] === undefined + ? null + : String(row["generatorAgentId"]), + provenance: String(row["provenance"] ?? "unknown"), + occurredAt: String(row["occurredAt"] ?? ""), + createdAt: String(row["createdAt"] ?? ""), + accessTags: Array.isArray(row["accessTags"]) + ? (row["accessTags"] as unknown[]).map(String) + : [], + }; + }); + const next = + o["nextCursor"] === null || o["nextCursor"] === undefined + ? null + : Number(o["nextCursor"]); + return { entries, nextCursor: next }; +} + +/** + * One distill tick: pull feed → host distill → write claims → return new cursor. + * Persist `nextCursor` after the tick succeeds (including poison advances). + */ +export async function runDistillTick( + args: RunDistillTickArgs, +): Promise { + const generatorAgentId = + args.generatorAgentId ?? RESIDENT_DISTILLER_AGENT_ID; + const after = args.after ?? 0; + + const raw = await args.client.feed( + { + after, + ...(args.limit !== undefined ? { limit: args.limit } : {}), + excludeGenerator: generatorAgentId, + }, + args.signal, + ); + const page = parseFeedPage(raw); + + let wrote = 0; + let skipped = 0; + let poisoned = 0; + const claims: DistilledClaimWrite[] = []; + + for (const entry of page.entries) { + if (!shouldProcessFeedEntry(entry, generatorAgentId)) { + skipped += 1; + continue; + } + let outcome: DistillOutcome; + try { + outcome = await args.distill(entry); + } catch (err) { + // Fail-soft: treat unexpected throw as poison so the cursor still advances. + poisoned += 1; + void err; + continue; + } + if (outcome.action === "skip") { + skipped += 1; + continue; + } + if (outcome.action === "poison") { + poisoned += 1; + continue; + } + + const claim = buildDistilledClaim({ + title: outcome.title, + text: outcome.text, + sourceAccessTags: entry.accessTags, + derivedFromVersionIds: [entry.versionId], + generatorAgentId, + ...(outcome.kind !== undefined ? { kind: outcome.kind } : {}), + ...(outcome.temporalClass !== undefined + ? { temporalClass: outcome.temporalClass } + : {}), + }); + await args.client.add(claim, args.signal); + claims.push(claim); + wrote += 1; + } + + const nextCursor = + page.nextCursor !== null && page.nextCursor !== undefined + ? page.nextCursor + : after; + + return { + nextCursor, + processed: page.entries.length, + wrote, + skipped, + poisoned, + claims, + }; +} diff --git a/src/distiller/workflow.test.ts b/src/distiller/workflow.test.ts new file mode 100644 index 0000000..663269c --- /dev/null +++ b/src/distiller/workflow.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "bun:test"; + +import { + RESIDENT_DISTILLER_AGENT_ID, + RESIDENT_DISTILLER_WORKFLOW_ID, +} from "./constants.ts"; +import { createResidentDistiller } from "./workflow.ts"; + +describe("createResidentDistiller", () => { + it("returns a schedule workflow with memory tools on the agent", () => { + const { workflow, agent, generatorAgentId } = createResidentDistiller({ + inference: { + sources: [{ provider: "openai", model: "gpt-4.1-mini" }], + }, + }); + + expect(generatorAgentId).toBe(RESIDENT_DISTILLER_AGENT_ID); + expect(workflow.id).toBe(RESIDENT_DISTILLER_WORKFLOW_ID); + expect(workflow.triggers).toEqual([ + { type: "schedule", cron: "*/5 * * * *" }, + ]); + expect(agent.id).toBe(RESIDENT_DISTILLER_AGENT_ID); + expect(agent.toolFactories.length).toBeGreaterThanOrEqual(3); + expect(agent.systemPrompt).toContain("memory_feed"); + expect(agent.systemPrompt).toContain(RESIDENT_DISTILLER_AGENT_ID); + }); + + it("allows cron and id overrides", () => { + const { workflow, generatorAgentId } = createResidentDistiller({ + id: "my-distiller", + agentId: "my-agent", + cron: "0 * * * *", + inference: { + sources: [{ provider: "openai", model: "gpt-4.1-mini" }], + }, + }); + expect(workflow.id).toBe("my-distiller"); + expect(generatorAgentId).toBe("my-agent"); + expect(workflow.triggers[0]).toEqual({ + type: "schedule", + cron: "0 * * * *", + }); + }); +}); diff --git a/src/distiller/workflow.ts b/src/distiller/workflow.ts new file mode 100644 index 0000000..312f830 --- /dev/null +++ b/src/distiller/workflow.ts @@ -0,0 +1,116 @@ +/** + * Ready-to-deploy resident distiller workflow for Interchange hosts. + * + * ```ts + * import { createResidentDistiller } from "@corbits/memory/distiller"; + * import { memoryAdd, memoryFeed, memorySearch } from "@corbits/memory/tools"; + * + * const workflow = createResidentDistiller({ + * inference: { sources: [{ provider: "openai", model: "gpt-4.1-mini" }] }, + * }); + * // deploy with host workflow-deploy + env: memoryBaseUrl, memoryTenantId, memoryAuthToken + * ``` + */ +import { + defineAgent, + type AgentDefinition, + type AnnotatedToolFactory, + type BaseEnv, + type InferencePreference, +} from "@intx/agent"; +import { defineWorkflow, type WorkflowDefinition } from "@intx/workflow"; + +import { memoryAdd } from "../tools/add.ts"; +import { memoryFeed } from "../tools/feed.ts"; +import { memorySearch } from "../tools/search.ts"; +import { + RESIDENT_DISTILLER_AGENT_ID, + RESIDENT_DISTILLER_CRON_DEFAULT, + RESIDENT_DISTILLER_WORKFLOW_ID, +} from "./constants.ts"; + +const DEFAULT_SYSTEM_PROMPT = `You are the resident memory distiller for a Corbits tenant. + +On each run: +1. Call memory_feed with after=, exclude_generator=${RESIDENT_DISTILLER_AGENT_ID}. +2. For each entry worth promoting to a durable claim: + - Classify (event / deadline / state / lesson) and gate junk / action-only noise. + - Write a concise claim via memory_add with: + generator_agent_id=${RESIDENT_DISTILLER_AGENT_ID} + provenance=inferred + lineage_class=derived + derived_from=[entry.versionId] + access_tags=entry.accessTags (copy exactly — never add broader tags) +3. Fail soft on poison entries: skip them and still advance past the page. +4. Remember the page nextCursor for the next run (host state / your notes). + +Never re-distill your own writes. Prefer few high-quality claims over many low-value ones. +When searching for corroboration, use memory_search and respect attribution (stated vs inferred).`; + +export type CreateResidentDistillerOpts = { + /** Workflow id (default resident-memory-distiller). */ + id?: string; + /** Agent id / generatorAgentId (default resident-distiller). */ + agentId?: string; + /** Cron schedule (default every 5 minutes). */ + cron?: string; + /** Host inference preferences (required for deploy hashing). */ + inference: { sources: readonly InferencePreference[] }; + /** Override system prompt. */ + systemPrompt?: string; + /** + * Extra tool factories beyond memory_feed / memory_add / memory_search. + * Default tools are always included first. + */ + extraTools?: readonly AnnotatedToolFactory[]; + /** Optional agent description. */ + description?: string; +}; + +export type ResidentDistiller = { + workflow: WorkflowDefinition; + agent: AgentDefinition; + generatorAgentId: string; +}; + +/** + * Build a schedule-triggered workflow + agent preloaded with memory tools. + * Host supplies inference sources and deploys with memory* env credentials. + */ +export function createResidentDistiller( + opts: CreateResidentDistillerOpts, +): ResidentDistiller { + const generatorAgentId = opts.agentId ?? RESIDENT_DISTILLER_AGENT_ID; + const tools = [ + memoryFeed, + memoryAdd, + memorySearch, + ...(opts.extraTools ?? []), + ] as AnnotatedToolFactory[]; + + const agent = defineAgent({ + id: generatorAgentId, + description: + opts.description ?? + "Resident memory distiller — feed → classify → claim write", + systemPrompt: opts.systemPrompt ?? DEFAULT_SYSTEM_PROMPT, + tools, + capabilities: ["memory:search", "memory:add"], + inference: opts.inference, + tags: { + role: "resident-distiller", + package: "@corbits/memory", + }, + }); + + const workflow = defineWorkflow({ + id: opts.id ?? RESIDENT_DISTILLER_WORKFLOW_ID, + trigger: { + type: "schedule", + cron: opts.cron ?? RESIDENT_DISTILLER_CRON_DEFAULT, + }, + agent, + }); + + return { workflow, agent, generatorAgentId }; +} diff --git a/src/http-bodies.ts b/src/http-bodies.ts index 6ad1291..d4ea242 100644 --- a/src/http-bodies.ts +++ b/src/http-bodies.ts @@ -25,6 +25,16 @@ export const AddRequest = type({ text: "string >= 1", "access_tags?": "string[]", "share?": ShareBody, + "kind?": "string", + /** Distiller / agent identity on the written version (loop-safety + attribution). */ + "generator_agent_id?": "string >= 1", + "provenance?": "'stated'|'inferred'|'unknown'", + "lineage_class?": "'native'|'imported'|'derived'", + "temporal_class?": "'event'|'deadline'|'state'|'lesson'", + /** Source version ids this claim is derived from (minted as derived_from edges). */ + "derived_from?": "string[]", + "valid_from?": "string", + "valid_until?": "string", }); export type AddRequest = typeof AddRequest.infer; diff --git a/src/index.ts b/src/index.ts index 994517e..434dc5d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,8 +6,10 @@ * `tenantId` in-process. Authz is the host grant store — this package * authenticates nothing itself. * - * Inference is host-owned and ephemeral (call your model, then add/search). - * Core does not mount an ingest agent or bake LLM into the write path. + * Distiller is first-class: `createResidentDistiller` / `runDistillTick` from + * `@corbits/memory/distiller` (or re-exported below). Inference stays host- + * injected; the package ships the workflow + tick helpers so apps opt in + * with a few lines. */ import type { Hono } from "hono"; import { createRequireGrant, type TenantEnv } from "@intx/hub-api"; @@ -136,6 +138,28 @@ export { type RetentionMutationResult, } from "./services/retention.ts"; +// Resident distiller (CL-5869) — also `@corbits/memory/distiller` +export { + RESIDENT_DISTILLER_AGENT_ID, + RESIDENT_DISTILLER_CRON_DEFAULT, + RESIDENT_DISTILLER_WORKFLOW_ID, + buildDistilledClaim, + createResidentDistiller, + resolveNextCursor, + runDistillTick, + shouldProcessFeedEntry, + type BuildDistilledClaimArgs, + type CreateResidentDistillerOpts, + type DistillOutcome, + type DistillTickFeedEntry, + type DistillTickPage, + type DistillTickResult, + type DistilledClaimWrite, + type FeedEntryLike, + type ResidentDistiller, + type RunDistillTickArgs, +} from "./distiller/index.ts"; + // Corroboration / living relevancy (CL-5867) export { corroborationFactor, diff --git a/src/memory.ts b/src/memory.ts index d7bc9e8..037dd7f 100644 --- a/src/memory.ts +++ b/src/memory.ts @@ -174,6 +174,19 @@ export type MemoryAddParams = MemoryIdentity & { */ share?: ShareSugar; attributes?: Record; + /** + * Claim / distiller identity (optional). When `generatorAgentId` is set the + * version is written as agent-authored so the capture feed can exclude it + * via `excludeGenerator` (loop-safety). + */ + generatorAgentId?: string; + provenance?: "stated" | "inferred" | "unknown"; + lineageClass?: "native" | "imported" | "derived"; + temporalClass?: "event" | "deadline" | "state" | "lesson"; + /** Source version ids → `derived_from` edges on the new version. */ + derivedFrom?: string[]; + validFrom?: string; + validUntil?: string; }; export type MemoryAddResult = { @@ -245,6 +258,8 @@ export type MemoryFeedEntry = { provenance: string; occurredAt: string; createdAt: string; + /** Grant-pattern tags for claim writes (≤ source access). */ + accessTags: string[]; }; export type MemoryFeedResult = { @@ -827,6 +842,27 @@ function createPlaneFromStore( : {}), ...(params.adapter !== undefined ? { adapter: params.adapter } : {}), ...(params.kind !== undefined ? { kind: params.kind } : {}), + ...(params.generatorAgentId !== undefined + ? { generatorAgentId: params.generatorAgentId } + : {}), + ...(params.provenance !== undefined + ? { provenance: params.provenance } + : {}), + ...(params.lineageClass !== undefined + ? { lineageClass: params.lineageClass } + : {}), + ...(params.temporalClass !== undefined + ? { temporalClass: params.temporalClass } + : {}), + ...(params.derivedFrom !== undefined + ? { derivedFrom: params.derivedFrom } + : {}), + ...(params.validFrom !== undefined + ? { validFrom: params.validFrom } + : {}), + ...(params.validUntil !== undefined + ? { validUntil: params.validUntil } + : {}), }); // Share materialization (CL-5873): stamp document-scoped tag + write @@ -1181,6 +1217,23 @@ function createEngineDocumentStore(config: MemoryConfig): { `memory:${params.tenantId}:${crypto.randomUUID()}`; const accessTags = params.accessTags ?? [ownerTag(params.principalId)]; + const generatorAgentId = params.generatorAgentId?.trim() || undefined; + const actor = generatorAgentId + ? { + kind: "agent" as const, + agentId: generatorAgentId, + principalId: params.principalId, + } + : { kind: "human" as const, principalId: params.principalId }; + + const edges = + params.derivedFrom && params.derivedFrom.length > 0 + ? params.derivedFrom.map((versionId) => ({ + rel: "derived_from" as const, + to: { type: "version" as const, ref: versionId }, + })) + : undefined; + const captureResult = await captureDocument(deps, { tenantId: params.tenantId, adapter, @@ -1192,11 +1245,31 @@ function createEngineDocumentStore(config: MemoryConfig): { accessTags, entityHints: [], chunks: [{ ordinal: 0, text: params.text }], - actor: { kind: "human", principalId: params.principalId }, + actor, contentHash: "", // recomputed canonically in adapt-and-plan ...(params.attributes !== undefined ? { attributes: params.attributes } : {}), + ...(params.provenance !== undefined + ? { provenance: params.provenance } + : generatorAgentId + ? { provenance: "inferred" as const } + : {}), + ...(params.lineageClass !== undefined + ? { lineageClass: params.lineageClass } + : generatorAgentId + ? { lineageClass: "derived" as const } + : {}), + ...(params.temporalClass !== undefined + ? { temporalClass: params.temporalClass } + : {}), + ...(params.validFrom !== undefined + ? { validFrom: params.validFrom } + : {}), + ...(params.validUntil !== undefined + ? { validUntil: params.validUntil } + : {}), + ...(edges !== undefined ? { edges } : {}), }, }); return { @@ -1313,6 +1386,7 @@ function createEngineDocumentStore(config: MemoryConfig): { provenance: e.provenance, occurredAt: e.occurredAt, createdAt: e.createdAt, + accessTags: e.accessTags, })); const nextCursor = entries.length > 0 ? entries[entries.length - 1]!.feedSeq : null; diff --git a/src/ports/types.ts b/src/ports/types.ts index 399330c..65e0f72 100644 --- a/src/ports/types.ts +++ b/src/ports/types.ts @@ -35,6 +35,14 @@ export type DocumentStoreAddParams = { adapter?: string; /** Document kind (default engine store uses `"note"`). */ kind?: string; + /** Claim / distiller fields — engine store only; vendors may ignore. */ + generatorAgentId?: string; + provenance?: "stated" | "inferred" | "unknown"; + lineageClass?: "native" | "imported" | "derived"; + temporalClass?: "event" | "deadline" | "state" | "lesson"; + derivedFrom?: string[]; + validFrom?: string; + validUntil?: string; }; export type DocumentStoreSearchParams = { @@ -132,6 +140,8 @@ export type DocumentStoreFeedEntry = { provenance: string; occurredAt: string; createdAt: string; + /** Grant-pattern tags — distiller copies onto claims (never widen). */ + accessTags: string[]; }; export type DocumentStoreFeedResult = { diff --git a/src/routes/add.ts b/src/routes/add.ts index fc99b50..f484175 100644 --- a/src/routes/add.ts +++ b/src/routes/add.ts @@ -64,6 +64,28 @@ export function mountAddRoute(app: Hono, deps: RouteDeps): void { principalId: subjectId, ...(accessTags !== undefined ? { accessTags } : {}), ...(share !== undefined ? { share } : {}), + ...(body.kind !== undefined ? { kind: body.kind } : {}), + ...(body.generator_agent_id !== undefined + ? { generatorAgentId: body.generator_agent_id } + : {}), + ...(body.provenance !== undefined + ? { provenance: body.provenance } + : {}), + ...(body.lineage_class !== undefined + ? { lineageClass: body.lineage_class } + : {}), + ...(body.temporal_class !== undefined + ? { temporalClass: body.temporal_class } + : {}), + ...(body.derived_from !== undefined + ? { derivedFrom: body.derived_from } + : {}), + ...(body.valid_from !== undefined + ? { validFrom: body.valid_from } + : {}), + ...(body.valid_until !== undefined + ? { validUntil: body.valid_until } + : {}), }); return c.json({ documentId: result.documentId, versionId: result.versionId }); } catch (err) { diff --git a/src/routes/feed.ts b/src/routes/feed.ts index a8fb88f..3f64abd 100644 --- a/src/routes/feed.ts +++ b/src/routes/feed.ts @@ -22,6 +22,7 @@ const FeedResponse = type({ provenance: "string", occurredAt: "string", createdAt: "string", + accessTags: "string[]", }).array(), nextCursor: "number|null", }); diff --git a/src/tools/add.ts b/src/tools/add.ts index 552341f..82b4c8e 100644 --- a/src/tools/add.ts +++ b/src/tools/add.ts @@ -12,6 +12,30 @@ function parseAddArgs(args: Record): MemoryAddBody { if (parsed.access_tags !== undefined) { body.access_tags = parsed.access_tags; } + if (parsed.kind !== undefined) { + body.kind = parsed.kind; + } + if (parsed.generator_agent_id !== undefined) { + body.generator_agent_id = parsed.generator_agent_id; + } + if (parsed.provenance !== undefined) { + body.provenance = parsed.provenance; + } + if (parsed.lineage_class !== undefined) { + body.lineage_class = parsed.lineage_class; + } + if (parsed.temporal_class !== undefined) { + body.temporal_class = parsed.temporal_class; + } + if (parsed.derived_from !== undefined) { + body.derived_from = parsed.derived_from; + } + if (parsed.valid_from !== undefined) { + body.valid_from = parsed.valid_from; + } + if (parsed.valid_until !== undefined) { + body.valid_until = parsed.valid_until; + } if (parsed.share !== undefined) { // Rebuild share field-by-field — arktype keeps undeclared nested keys. const share: NonNullable = {}; @@ -33,15 +57,17 @@ function parseAddArgs(args: Record): MemoryAddBody { * Installable tool: POST /api/tenants/:tenantId/memory/add. * * Tenant and auth come from env (`memoryTenantId`, `memoryAuthToken`); - * model args never carry identity. + * model args never carry identity. Distiller claims pass + * `generator_agent_id` + `derived_from` for loop-safety and lineage. */ export const memoryAdd = defineMemoryHttpTool({ id: "@corbits/memory/add", name: "memory_add", description: "Store a note in tenant memory. Returns { documentId, versionId }. " + - "Identity is the authenticated principal on the hub; do not " + - "pass tenant or principal ids.", + "For distilled claims set generator_agent_id, provenance=inferred, " + + "lineage_class=derived, and derived_from source version ids. " + + "Identity is the authenticated principal on the hub.", inputSchema: { type: "object", properties: { @@ -57,7 +83,46 @@ export const memoryAdd = defineMemoryHttpTool({ type: "array", items: { type: "string" }, description: - "Optional grant-pattern tags controlling document visibility", + "Optional grant-pattern tags controlling document visibility " + + "(distiller: copy from source feed entry, never widen)", + }, + kind: { + type: "string", + description: "Document kind (default note)", + }, + generator_agent_id: { + type: "string", + description: + "Agent id that authored this version (e.g. resident-distiller). " + + "Enables feed excludeGenerator loop-safety.", + }, + provenance: { + type: "string", + enum: ["stated", "inferred", "unknown"], + description: "How content was obtained (inferred for distilled claims)", + }, + lineage_class: { + type: "string", + enum: ["native", "imported", "derived"], + description: "Data lineage (derived for distilled claims)", + }, + temporal_class: { + type: "string", + enum: ["event", "deadline", "state", "lesson"], + description: "Temporal ranking class", + }, + derived_from: { + type: "array", + items: { type: "string" }, + description: "Source version ids this claim is derived from", + }, + valid_from: { + type: "string", + description: "Optional validity start (ISO)", + }, + valid_until: { + type: "string", + description: "Optional validity end (ISO)", }, share: { type: "object", diff --git a/src/tools/client.ts b/src/tools/client.ts index 37cef5e..02b44e1 100644 --- a/src/tools/client.ts +++ b/src/tools/client.ts @@ -26,6 +26,14 @@ export type MemoryHttpClient = { add(body: MemoryAddBody, signal?: AbortSignal): Promise; search(body: MemorySearchBody, signal?: AbortSignal): Promise; list(limit?: number, signal?: AbortSignal): Promise; + feed( + opts?: { + after?: number; + limit?: number; + excludeGenerator?: string; + }, + signal?: AbortSignal, + ): Promise; }; /** Cap hub error text embedded in tool errors (avoid huge/secret-ish dumps). */ @@ -127,6 +135,23 @@ export function createMemoryHttpClient( ...(signal !== undefined ? { signal } : {}), }); }, + feed(opts, signal) { + const params = new URLSearchParams(); + if (opts?.after !== undefined) { + params.set("after", String(opts.after)); + } + if (opts?.limit !== undefined) { + params.set("limit", String(opts.limit)); + } + if (opts?.excludeGenerator !== undefined) { + params.set("exclude_generator", opts.excludeGenerator); + } + const qs = params.toString(); + return request(`/feed${qs ? `?${qs}` : ""}`, { + method: "GET", + ...(signal !== undefined ? { signal } : {}), + }); + }, }; } diff --git a/src/tools/feed.ts b/src/tools/feed.ts new file mode 100644 index 0000000..795b8e4 --- /dev/null +++ b/src/tools/feed.ts @@ -0,0 +1,78 @@ +import { type } from "arktype"; + +import { parseWithArk } from "../http-bodies.ts"; +import { defineMemoryHttpTool } from "./install.ts"; + +const FeedArgs = type({ + "after?": "number.integer >= 0", + "limit?": "1 <= number.integer <= 100", + "exclude_generator?": "string", +}); + +function coerceFeedArgs(args: Record): Record { + const out = { ...args }; + for (const key of ["after", "limit"] as const) { + const raw = out[key]; + if (typeof raw === "string" && raw.trim() !== "") { + const n = Number(raw); + if (Number.isFinite(n)) out[key] = n; + } + } + return out; +} + +/** + * Installable tool: GET /api/tenants/:tenantId/memory/feed. + * + * Distiller pull surface — always pass exclude_generator matching the + * writer's generator_agent_id so the agent never re-consumes its own claims. + */ +export const memoryFeed = defineMemoryHttpTool({ + id: "@corbits/memory/feed", + name: "memory_feed", + description: + "Pull new memory versions after a cursor (capture feed). " + + "Returns { entries, nextCursor }. Always set exclude_generator to your " + + "generator_agent_id (e.g. resident-distiller) for loop-safety. " + + "Copy accessTags from each entry onto distilled writes — never widen.", + inputSchema: { + type: "object", + properties: { + after: { + type: "integer", + minimum: 0, + description: "Exclusive cursor (feed_seq > after). Default 0.", + }, + limit: { + type: "integer", + minimum: 1, + maximum: 100, + description: "Page size (1–100, default 50)", + }, + exclude_generator: { + type: "string", + description: + "Skip versions written by this generator_agent_id (loop-safety)", + }, + }, + additionalProperties: false, + }, + async handle(client, args, signal) { + const parsed = parseWithArk( + FeedArgs, + coerceFeedArgs(args), + "memory_feed", + ); + const result = await client.feed( + { + ...(parsed.after !== undefined ? { after: parsed.after } : {}), + ...(parsed.limit !== undefined ? { limit: parsed.limit } : {}), + ...(parsed.exclude_generator !== undefined + ? { excludeGenerator: parsed.exclude_generator } + : {}), + }, + signal, + ); + return JSON.stringify(result); + }, +}); diff --git a/src/tools/index.ts b/src/tools/index.ts index 0249836..4fca0dd 100644 --- a/src/tools/index.ts +++ b/src/tools/index.ts @@ -9,6 +9,7 @@ export { memoryAdd } from "./add.ts"; export { memorySearch } from "./search.ts"; export { memoryList } from "./list.ts"; +export { memoryFeed } from "./feed.ts"; export { createMemoryHttpClient, MEMORY_TOOL_ENV_KEYS, @@ -16,3 +17,4 @@ export { type MemoryHttpConfig, type MemoryToolEnv, } from "./client.ts"; + From e09106bf0e9f2fb239f87d3e84a7b1f5bec4ba92 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 10 Aug 2026 11:09:42 -0700 Subject: [PATCH 13/19] docs: distiller layout + integration checklist IMPLEMENTATION.md lists distiller/ and tools/; DISTILLER.md adds a manual dev-hub integration checklist for CL-5869. --- IMPLEMENTATION.md | 10 +++++++++- docs/DISTILLER.md | 12 ++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md index 5cc4b93..75c7b6d 100644 --- a/IMPLEMENTATION.md +++ b/IMPLEMENTATION.md @@ -8,7 +8,7 @@ and wire shapes. For the "why standalone" / boundaries story, read ``` src/ - index.ts # createMemory / registerMemoryRoutes + index.ts # createMemory / registerMemoryRoutes + distiller re-exports mount-config.ts # MemoryConfig + loadMemoryConfig() — the mount config config.ts # EngineConfig — the core vector-plane config (db + embed + rerank) @@ -23,6 +23,13 @@ src/ deps.ts # RouteDeps, caller(c) (context identity), grantGuard add.ts, search.ts, list.ts, feed.ts + tools/ # Interchange defineTool factories (HTTP clients) + add.ts, search.ts, list.ts, feed.ts, client.ts, install.ts + distiller/ # Resident distiller (CL-5869) — workflow + tick helpers + index.ts # createResidentDistiller, runDistillTick, buildDistilledClaim + workflow.ts # defineWorkflow + defineAgent with memory tools + tick.ts # imperative distill tick (host injects distill()) + claim.ts # pure claim body / gate / cursor helpers db/ schema.ts # Drizzle table defs (memory.* schema) client.ts # createDb(config) -> { db (drizzle), sql (raw postgres-js) } @@ -32,6 +39,7 @@ src/ timeline.ts # listTimelineEvents — durable recent docs + grant-tag filter transform.ts # transform_config CRUD + runTransform (replay) feed.ts # capture feed cursor pull (CL-5868) + retention.ts # deprecate / tombstone / sweep ephemeral (CL-5871) share-grants.ts # peer grant materialization on share (CL-5873) core/ # framework-agnostic (chunking, embed/rerank, merge, schemas) # DocumentStore adapters / tools live as sibling packages (not in this tree): diff --git a/docs/DISTILLER.md b/docs/DISTILLER.md index 1416034..0da6735 100644 --- a/docs/DISTILLER.md +++ b/docs/DISTILLER.md @@ -85,6 +85,18 @@ Minimum capabilities for the distiller principal: Copy `accessTags` from each feed entry onto writes — never mint broader tags. +## Manual integration checklist (dev hub) + +When a hub + memory mount is available: + +1. Deploy `createResidentDistiller({ inference })` (or cron `runDistillTick`). +2. Capture ~20 mixed notes (human + agent) via `memory_add`. +3. Confirm feed pages with `exclude_generator=resident-distiller`. +4. Confirm distilled claims appear with `attribution.provenance=inferred`, + `generatorAgentId=resident-distiller`, and `derivedFrom` set. +5. Restart mid-cursor — next tick resumes from persisted `nextCursor`. +6. Inject one poison entry — cursor still advances; no stuck feed. + ## Out of scope - Host deploy pipeline / secrets From dd36f680cc9e760a331544aa301a01928e31a5e1 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 10 Aug 2026 12:17:11 -0700 Subject: [PATCH 14/19] =?UTF-8?q?docs:=20lock=20add=20=E2=86=92=20ingest?= =?UTF-8?q?=20elements=20=E2=86=92=20process=20as=20default=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Product narrative: one host pipeline; pull feed + resident distiller are optional multi-writer/backfill process helpers, not primary ingest. --- ARCHITECTURE.md | 11 ++++- CHANGELOG.md | 4 ++ PRODUCT.md | 43 ++++++++++++++----- docs/DISTILLER.md | 103 ++++++++++++++++++++++++---------------------- docs/FEED.md | 18 ++++++-- 5 files changed, 115 insertions(+), 64 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 0ddd336..95d91b3 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -22,7 +22,11 @@ talks to its DocumentStore. No second server. ## Product path ``` -tools / ingestion → /api/tenants/:tenantId/memory/* → Memory plane → DocumentStore +add → ingest elements (store/chunk/embed) → process (optional, host) +``` + +``` +tools / host ingest workflow → /api/tenants/:tenantId/memory/* → Memory plane → DocumentStore ↑ Interchange auth + principal + grants ``` @@ -31,6 +35,11 @@ Mount is intentionally small. The host already has `app`, grants, and principal middleware; memory only needs to be handed those and the vector config (or an injected store). +**Ingest elements** run on the default store inside `add` (raw capture, chunks, +edges, embed). **Process** (claims, LLM link/classify) is host-owned inference, +preferably in the same workflow body as the add. Capture **feed** + distiller +helpers are optional multi-writer / backfill — not the primary path. + ## Boundaries - **Runtime**: Bun + Hono, mounted on the host app. **DB**: own pgvector diff --git a/CHANGELOG.md b/CHANGELOG.md index fe46dea..bc318b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **Product narrative:** default path is **add → ingest elements → process** + (one host pipeline). Pull feed + `createResidentDistiller` are optional + multi-writer / backfill process helpers, not the primary ingest story. + See `PRODUCT.md`, `docs/DISTILLER.md`, `docs/FEED.md`. - **Breaking:** Postgres schema renamed from `knowledge` to **`memory`**. Fresh installs only — drop/recreate the old schema (or rename) on existing DBs. Citation `open.type` is now `"memory"`. diff --git a/PRODUCT.md b/PRODUCT.md index b0a76e0..105f3cc 100644 --- a/PRODUCT.md +++ b/PRODUCT.md @@ -4,9 +4,27 @@ Memory for Interchange hubs: durable documents, hybrid search, recent list. **You mount it on the hub (~5 lines). That exposes protected routes. Agents and ingestion modules call those routes.** Workbench and coding agents are -clients — not owners of auth. Inference stays host-injected; the package ships -add/search/list **and** an optional resident distiller so apps can opt into -continuous distillation with a few lines (`docs/DISTILLER.md`). +clients — not owners of auth. Inference stays host-injected. + +## Default pipeline (locked) + +```text +add → ingest elements → process (optional) +``` + +| Stage | Meaning | Where | +| --- | --- | --- | +| **add** | Something arrives (agent tool, host job, webhook body) | Caller → `memory.add` / `POST …/memory/add` | +| **ingest elements** | Normalize → raw capture → chunks / edges → embed → search-ready | Default `DocumentStore` capture path (sync on `add`) | +| **process** | Optional brain work: classify, claims, links, forget | Host workflow / injected inference — same run as ingest when possible | + +Preferred host shape: **one ingest workflow** receives the event, calls `add` +(ingest elements), then runs process steps in the same body (or a child step). +No pull feed required on that path — the workflow already has the payload. + +**Pull feed + resident distiller** are optional: multi-writer backfill, replay, +or polish when other code also `add`s outside the ingest workflow. See +`docs/DISTILLER.md` and `docs/FEED.md`. ## Shape (locked) @@ -20,22 +38,22 @@ never creates one; it mounts onto yours. | `runMemoryMigrations(url)` | Apply pgvector schema | | `registerMemoryRoutes` | Low-level HTTP only (optional) | | `@corbits/memory/tools` | Interchange tools (`memory_add` / `search` / `list` / `feed`) | -| `@corbits/memory/distiller` | `createResidentDistiller` workflow + `runDistillTick` | +| `@corbits/memory/distiller` | Optional process helpers: `runDistillTick`, `createResidentDistiller` | ### Verbs | Method | HTTP | Grant | Meaning | | --- | --- | --- | --- | -| `add` | `POST /api/tenants/:tenantId/memory/add` | `memory:add` | Capture a document | +| `add` | `POST /api/tenants/:tenantId/memory/add` | `memory:add` | Ingest: capture + derive (chunk/embed on default store) | | `search` | `POST /api/tenants/:tenantId/memory/search` | `memory:search` | Hybrid retrieval (+ optional live sources); hits may include additive `attribution` | | `list` | `GET /api/tenants/:tenantId/memory/list` | `memory:search` | Recent documents for the principal | -| `feed` | `GET /api/tenants/:tenantId/memory/feed` | `memory:search` | Cursor pull of new live versions (distiller) | +| `feed` | `GET /api/tenants/:tenantId/memory/feed` | `memory:search` | Cursor pull of new live versions (optional multi-writer / backfill) | Engine-only plane helpers (no HTTP yet): transform/replay, retention (`deprecateVersion` / `tombstoneDocument` / … — see `docs/RETENTION.md`), -share-grant materialization. Distiller is first-class: +share-grant materialization. Process helpers: `createResidentDistiller` / `runDistillTick` (`docs/DISTILLER.md`) — host -injects inference; package ships the workflow + tick helpers. +injects inference; not the default ingest path. Identity is always **`principalId` + `tenantId`** on the plane. HTTP routes never take body identity — they read `c.get("principal")` from Interchange @@ -44,7 +62,7 @@ context. ### How it is used ``` -Agent / ingestion module +Agent / host ingest workflow │ tool call or host worker │ → POST|GET /api/tenants/:tenantId/memory/* │ authenticated by Interchange (session | API key | MCP OAuth) @@ -67,8 +85,10 @@ Agent / ingestion module workflow with env credentials (`memoryBaseUrl`, `memoryTenantId`, `memoryAuthToken`). Tools HTTP-call the mounted routes; identity is the hub-authenticated principal. OpenAPI→MCP remains an optional host bridge. -3. **Ingestion** — host modules (webhooks, batch jobs) call the routes or the - returned plane with a resolved principal. +3. **Ingestion** — preferred: one host workflow (or module) does + **add → ingest elements → process**. Mechanical ingest is inside `add` on + the default store; process (claims / links) is host-injected inference in + the same pipeline when you want a company brain. ### Ports @@ -85,6 +105,7 @@ stores, Linear tools. Core never imports vendor SDKs. - No auth, API keys, OAuth, webhooks, SPA, or standalone server in core. - No answer/generation endpoint — host owns inference. - Workbench is a client, not required. +- Core does not run the ingest workflow process — the host does. **Default durable store:** Postgres via `DATABASE_URL`, tables under the **`memory`** schema. When diff --git a/docs/DISTILLER.md b/docs/DISTILLER.md index 0da6735..ccc4cab 100644 --- a/docs/DISTILLER.md +++ b/docs/DISTILLER.md @@ -1,31 +1,40 @@ -# Resident distiller +# Process helpers (distiller) -First-class on-ramp so a Corbits app can add memory **and** keep it distilled -without a sibling package. +Default product path is **add → ingest elements → process** in one host +pipeline (`PRODUCT.md`). Mechanical ingest (raw → chunk → embed) already runs +inside `memory.add` on the default store. -## Quick start (Interchange workflow) +This package’s distiller exports are **optional process helpers** for when +you need LLM claim extraction **outside** that single pipeline: -```ts -import { createResidentDistiller } from "@corbits/memory/distiller"; -// or: import { createResidentDistiller } from "@corbits/memory"; +- multi-writer: other agents also `add` and you want a backfill worker +- replay / catch-up over a cursor +- fail-soft polish decoupled from the write that ingested the raw note -const { workflow, generatorAgentId } = createResidentDistiller({ - inference: { - sources: [{ provider: "openai", model: "gpt-4.1-mini" }], - }, - // optional: cron: "*/5 * * * *", id, agentId, systemPrompt, extraTools -}); +They are **not** the primary “how memory is ingested” story. + +## Preferred: process in the same ingest workflow -// Deploy `workflow` with host workflow-deploy. -// Env for tools: memoryBaseUrl, memoryTenantId, memoryAuthToken -// Grant the distiller principal: memory:search + memory:add (feed via search grant) +```text +onTrigger / host job + 1. receive source event (or fetch) + 2. memory_add // ingest elements + 3. process (optional) // claims / links — same body or child step ``` -The agent is preloaded with `memory_feed`, `memory_add`, and `memory_search`. -System prompt encodes loop-safety (`exclude_generator` = `generatorAgentId`), -access-tag copy (never widen), and fail-soft poison handling. +No feed cursor required: the workflow already has the payload. Host injects +inference; tools only need `memory_add` / `memory_search` (and grants). -## Quick start (imperative — any scheduler) +Helpers still useful in-process: + +| Export | Use | +| --- | --- | +| `buildDistilledClaim` | Wire body with `generator_agent_id`, `provenance=inferred`, `derived_from` | +| `RESIDENT_DISTILLER_AGENT_ID` | Stable generator id if you write claims | + +## Optional: multi-writer / backfill (`runDistillTick`) + +When other writers also `add`, drain new versions with the capture feed: ```ts import { runDistillTick } from "@corbits/memory/distiller"; @@ -57,48 +66,44 @@ cursor = result.nextCursor; // persist Inference is **always injected** (`distill` callback or host agent sources). The package never embeds a model. -## Helpers +## Optional: schedule workflow scaffold (`createResidentDistiller`) -| Export | Use | -| --- | --- | -| `buildDistilledClaim` | Wire body with `generator_agent_id`, `provenance=inferred`, `derived_from` | -| `shouldProcessFeedEntry` | Skip own generator writes (defense in depth) | -| `resolveNextCursor` | Fail-soft cursor advance after poison | -| `RESIDENT_DISTILLER_AGENT_ID` | Default `"resident-distiller"` | +Scaffold for hosts that still want a deployed agent with memory tools + +system prompt (loop-safety, access-tag copy). Prefer wiring **process next to +add** in your ingest workflow; use this for backfill-style residency only. -## Substrate (already on the plane) +```ts +import { createResidentDistiller } from "@corbits/memory/distiller"; + +const { workflow, generatorAgentId } = createResidentDistiller({ + inference: { + sources: [{ provider: "openai", model: "gpt-4.1-mini" }], + }, +}); +// Deploy only if you need a multi-writer pull consumer — not default ingest. +``` + +## Substrate (plane) | Piece | Where | | --- | --- | -| Capture feed (exactly-once cursor) | `memory.feed` / `GET …/memory/feed` — [FEED.md](./FEED.md) | +| Ingest on add | capture path — raw + chunks + embed | +| Capture feed (cursor) | `memory.feed` — [FEED.md](./FEED.md) (backfill / multi-writer) | | Claim identity on add | `generator_agent_id`, `provenance`, `lineage_class`, `derived_from` | | Wire attribution on search | `SearchItem.attribution` | | Retention / forgetting | [RETENTION.md](./RETENTION.md) | -| Tools | `@corbits/memory/tools` — `memoryAdd`, `memoryFeed`, `memorySearch`, `memoryList` | - -## Grant manifest (host) - -Minimum capabilities for the distiller principal: - -- `memory:search` (covers feed pull under the same capability family hosts use today) -- `memory:add` (claim writes) - -Copy `accessTags` from each feed entry onto writes — never mint broader tags. +| Tools | `@corbits/memory/tools` | -## Manual integration checklist (dev hub) +## Grant manifest (process principal) -When a hub + memory mount is available: +- `memory:add` (claim or note writes) +- `memory:search` (corroboration + feed if using backfill) -1. Deploy `createResidentDistiller({ inference })` (or cron `runDistillTick`). -2. Capture ~20 mixed notes (human + agent) via `memory_add`. -3. Confirm feed pages with `exclude_generator=resident-distiller`. -4. Confirm distilled claims appear with `attribution.provenance=inferred`, - `generatorAgentId=resident-distiller`, and `derivedFrom` set. -5. Restart mid-cursor — next tick resumes from persisted `nextCursor`. -6. Inject one poison entry — cursor still advances; no stuck feed. +Copy `accessTags` from the source onto claim writes — never mint broader tags. ## Out of scope - Host deploy pipeline / secrets -- Push outbox (feed remains pull-only) +- Push outbox (optional later; not required if process is in-pipeline) +- Core-owned ingest workflow process (host owns that) - Automatic supports/contradicts edge minting beyond `derived_from` on add diff --git a/docs/FEED.md b/docs/FEED.md index e42d21b..2d86b18 100644 --- a/docs/FEED.md +++ b/docs/FEED.md @@ -1,7 +1,17 @@ # Capture feed -Stateless, cursorable pull of new **versions** for the resident distiller -(CL-5868). +Stateless, cursorable pull of new **versions** for **optional** multi-writer +backfill / process workers (CL-5868). + +**Default product path does not need this.** Prefer +**add → ingest elements → process** in one host pipeline (`PRODUCT.md`): the +workflow already has the payload, so no pull cursor. + +Use the feed when: + +- other agents or modules also `add` outside your ingest workflow +- you need catch-up / replay over a durable ordering key +- process is intentionally decoupled from the write (fail-soft polish) ## Phase 1 — pull (implemented) @@ -28,10 +38,12 @@ HTTP: `GET /api/tenants/:tenantId/memory/feed?after=&limit=&exclude_generator=` Post-commit outbox row keyed by `feed_seq` + host dispatcher that mails the deployment address with version ids. **Not implemented** in core. Phase 1 -`feed_seq` is the ordering key so Phase 2 is additive. +`feed_seq` is the ordering key so Phase 2 is additive. Only relevant if process +stays out-of-band from the writer. ## Non-goals - In-core cron or push dispatcher - Bypassing grant tags for “tenant brain” - Including replay generations in the default feed +- Replacing the default **add → ingest → process** host pipeline From cf4306d67b58c15a0a3bce1ebb275b1260d64f65 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 10 Aug 2026 16:53:52 -0700 Subject: [PATCH 15/19] feat: interchange.grantRequirements for installer discovery Static package.json + MEMORY_GRANT_REQUIREMENTS SSOT so host installers can learn memory:add / memory:search without executing the package. --- docs/DISTILLER.md | 11 ++++++- package.json | 16 +++++++++- src/distiller/workflow.ts | 4 ++- src/grant-requirements.test.ts | 54 ++++++++++++++++++++++++++++++++++ src/grant-requirements.ts | 49 ++++++++++++++++++++++++++++++ src/index.ts | 9 ++++++ src/tools/index.ts | 9 ++++++ 7 files changed, 149 insertions(+), 3 deletions(-) create mode 100644 src/grant-requirements.test.ts create mode 100644 src/grant-requirements.ts diff --git a/docs/DISTILLER.md b/docs/DISTILLER.md index ccc4cab..66f4138 100644 --- a/docs/DISTILLER.md +++ b/docs/DISTILLER.md @@ -96,10 +96,19 @@ const { workflow, generatorAgentId } = createResidentDistiller({ ## Grant manifest (process principal) +Installer discovery (not live grants): + +- `package.json` → `interchange.grantRequirements` +- typed SSOT: `MEMORY_GRANT_REQUIREMENTS` / `MEMORY_CAPABILITY_IDS` from + `@corbits/memory` (or `@corbits/memory/tools`) + +Minimum capabilities: + - `memory:add` (claim or note writes) - `memory:search` (corroboration + feed if using backfill) -Copy `accessTags` from the source onto claim writes — never mint broader tags. +Deploy materializes these onto the workflow principal. Copy `accessTags` from +the source onto claim writes — never mint broader tags. ## Out of scope diff --git a/package.json b/package.json index 0065269..963fe78 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,21 @@ "./distiller": "./src/distiller/index.ts" }, "interchange": { - "tools": "./src/tools/index.ts" + "tools": "./src/tools/index.ts", + "grantRequirements": [ + { + "resource": "memory", + "action": "add", + "source": "tenant", + "surfaces": ["tools", "distiller", "routes"] + }, + { + "resource": "memory", + "action": "search", + "source": "tenant", + "surfaces": ["tools", "distiller", "routes"] + } + ] }, "license": "LGPL-2.1-only", "type": "module", diff --git a/src/distiller/workflow.ts b/src/distiller/workflow.ts index 312f830..778f557 100644 --- a/src/distiller/workflow.ts +++ b/src/distiller/workflow.ts @@ -23,6 +23,7 @@ import { defineWorkflow, type WorkflowDefinition } from "@intx/workflow"; import { memoryAdd } from "../tools/add.ts"; import { memoryFeed } from "../tools/feed.ts"; import { memorySearch } from "../tools/search.ts"; +import { MEMORY_CAPABILITY_IDS } from "../grant-requirements.ts"; import { RESIDENT_DISTILLER_AGENT_ID, RESIDENT_DISTILLER_CRON_DEFAULT, @@ -95,7 +96,8 @@ export function createResidentDistiller( "Resident memory distiller — feed → classify → claim write", systemPrompt: opts.systemPrompt ?? DEFAULT_SYSTEM_PROMPT, tools, - capabilities: ["memory:search", "memory:add"], + capabilities: [...MEMORY_CAPABILITY_IDS], + inference: opts.inference, tags: { role: "resident-distiller", diff --git a/src/grant-requirements.test.ts b/src/grant-requirements.test.ts new file mode 100644 index 0000000..62aa248 --- /dev/null +++ b/src/grant-requirements.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { + MEMORY_CAPABILITY_IDS, + MEMORY_GRANT_REQUIREMENTS, +} from "./grant-requirements.ts"; + +describe("MEMORY_GRANT_REQUIREMENTS", () => { + test("covers add + search on memory resource", () => { + expect(MEMORY_GRANT_REQUIREMENTS.map((r) => r.action).sort()).toEqual([ + "add", + "search", + ]); + for (const r of MEMORY_GRANT_REQUIREMENTS) { + expect(r.resource).toBe("memory"); + expect(r.source).toBe("tenant"); + expect(r.surfaces).toContain("tools"); + expect(r.surfaces).toContain("distiller"); + } + }); + + test("capability ids are resource:action", () => { + expect([...MEMORY_CAPABILITY_IDS].sort()).toEqual([ + "memory:add", + "memory:search", + ]); + }); + + test("package.json interchange.grantRequirements stays in lockstep", () => { + const pkg = JSON.parse( + readFileSync(join(import.meta.dir, "..", "package.json"), "utf8"), + ) as { + interchange?: { + grantRequirements?: Array<{ + resource: string; + action: string; + source: string; + surfaces: string[]; + }>; + }; + }; + const fromPkg = pkg.interchange?.grantRequirements ?? []; + expect(fromPkg).toEqual( + MEMORY_GRANT_REQUIREMENTS.map((r) => ({ + resource: r.resource, + action: r.action, + source: r.source, + surfaces: [...r.surfaces], + })), + ); + }); +}); diff --git a/src/grant-requirements.ts b/src/grant-requirements.ts new file mode 100644 index 0000000..e52d530 --- /dev/null +++ b/src/grant-requirements.ts @@ -0,0 +1,49 @@ +/** + * Grant *requirements* for installers — not live grants. + * + * Mirrored under `package.json` → `interchange.grantRequirements` so a + * host installer can read npm metadata without executing code. The typed + * export is the in-repo SSOT; keep package.json in lockstep. + * + * Shape matches Interchange definition grant requirements + * (`resource` + `action` + `source`). Control plane materializes grants + * onto the workflow principal at deploy/launch. + */ + +export type MemoryGrantSource = "tenant" | "creator" | "invoker"; + +/** Package surfaces that need the requirement when installed. */ +export type MemoryGrantSurface = "tools" | "distiller" | "routes"; + +export type MemoryGrantRequirement = { + readonly resource: string; + readonly action: string; + /** Recommended authority source; installer/deploy may override. */ + readonly source: MemoryGrantSource; + readonly surfaces: readonly MemoryGrantSurface[]; +}; + +/** + * Minimum capability grants for memory tools / routes / process helpers. + * Document-tag access (`memory.doc:…`, `memory.space:…`) is separate and + * minted per document — not package install requirements. + */ +export const MEMORY_GRANT_REQUIREMENTS = [ + { + resource: "memory", + action: "add", + source: "tenant", + surfaces: ["tools", "distiller", "routes"], + }, + { + resource: "memory", + action: "search", + source: "tenant", + surfaces: ["tools", "distiller", "routes"], + }, +] as const satisfies readonly MemoryGrantRequirement[]; + +/** Compact `resource:action` form used on agent `capabilities` arrays. */ +export const MEMORY_CAPABILITY_IDS = MEMORY_GRANT_REQUIREMENTS.map( + (r) => `${r.resource}:${r.action}` as const, +); diff --git a/src/index.ts b/src/index.ts index 434dc5d..4e16314 100644 --- a/src/index.ts +++ b/src/index.ts @@ -62,6 +62,15 @@ export { LIST_LIMIT_MAX, } from "./memory.ts"; +// Installer discovery — grant *requirements* (not live grants) +export { + MEMORY_CAPABILITY_IDS, + MEMORY_GRANT_REQUIREMENTS, + type MemoryGrantRequirement, + type MemoryGrantSource, + type MemoryGrantSurface, +} from "./grant-requirements.ts"; + // Ports — pluggable storage and live sources export type { DocumentStore, diff --git a/src/tools/index.ts b/src/tools/index.ts index 4fca0dd..95d31fc 100644 --- a/src/tools/index.ts +++ b/src/tools/index.ts @@ -18,3 +18,12 @@ export { type MemoryToolEnv, } from "./client.ts"; +/** Re-export installer grant requirements (same as package root). */ +export { + MEMORY_CAPABILITY_IDS, + MEMORY_GRANT_REQUIREMENTS, + type MemoryGrantRequirement, + type MemoryGrantSource, + type MemoryGrantSurface, +} from "../grant-requirements.ts"; + From 4bf9d11f67fcca7081f5492c1535816af70e99d2 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Thu, 13 Aug 2026 23:19:05 -0700 Subject: [PATCH 16/19] fix: advance feed cursor past ACL-denied pages Post-filter used the last allowed feedSeq for nextCursor, so a fully denied page returned null and stalled consumers. Keep the raw page cursor; wire agentId into the distiller default system prompt. --- CHANGELOG.md | 7 +++++++ docs/FEED.md | 3 +++ src/distiller/workflow.test.ts | 4 +++- src/distiller/workflow.ts | 9 +++++---- src/memory.ts | 8 +++++--- src/services/feed.test.ts | 33 +++++++++++++++++++++++++++++++++ src/services/feed.ts | 17 +++++++++++++++++ 7 files changed, 73 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc318b5..6d944a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- Feed `nextCursor` advances past the examined raw page after grant-tag + post-filter (a fully denied page no longer stalls the consumer forever). +- Distiller default system prompt uses the configured `agentId` for + `exclude_generator` / `generator_agent_id` (not only the package default id). + ### Changed - **Product narrative:** default path is **add → ingest elements → process** diff --git a/docs/FEED.md b/docs/FEED.md index 2d86b18..e882c41 100644 --- a/docs/FEED.md +++ b/docs/FEED.md @@ -29,6 +29,9 @@ memory.feed({ tenantId, principalId, after?, limit?, excludeGenerator? }) - **Live generation only** — same rule as default search. - Capability: `memory` / `search` (same as list/retrieve). - Document access: grant-tag post-filter identical to search. +- **Cursor advances past the examined raw page**, even when the access filter + returns zero entries. Using the last *allowed* `feedSeq` would stall a + consumer on a fully denied page forever. Cursor storage is the **consumer's** job (workflow run state). diff --git a/src/distiller/workflow.test.ts b/src/distiller/workflow.test.ts index 663269c..f7f1956 100644 --- a/src/distiller/workflow.test.ts +++ b/src/distiller/workflow.test.ts @@ -26,7 +26,7 @@ describe("createResidentDistiller", () => { }); it("allows cron and id overrides", () => { - const { workflow, generatorAgentId } = createResidentDistiller({ + const { workflow, generatorAgentId, agent } = createResidentDistiller({ id: "my-distiller", agentId: "my-agent", cron: "0 * * * *", @@ -40,5 +40,7 @@ describe("createResidentDistiller", () => { type: "schedule", cron: "0 * * * *", }); + expect(agent.systemPrompt).toContain("my-agent"); + expect(agent.systemPrompt).not.toContain(RESIDENT_DISTILLER_AGENT_ID); }); }); diff --git a/src/distiller/workflow.ts b/src/distiller/workflow.ts index 778f557..b59656b 100644 --- a/src/distiller/workflow.ts +++ b/src/distiller/workflow.ts @@ -30,14 +30,15 @@ import { RESIDENT_DISTILLER_WORKFLOW_ID, } from "./constants.ts"; -const DEFAULT_SYSTEM_PROMPT = `You are the resident memory distiller for a Corbits tenant. +const DEFAULT_SYSTEM_PROMPT = (generatorAgentId: string) => + `You are the resident memory distiller for a Corbits tenant. On each run: -1. Call memory_feed with after=, exclude_generator=${RESIDENT_DISTILLER_AGENT_ID}. +1. Call memory_feed with after=, exclude_generator=${generatorAgentId}. 2. For each entry worth promoting to a durable claim: - Classify (event / deadline / state / lesson) and gate junk / action-only noise. - Write a concise claim via memory_add with: - generator_agent_id=${RESIDENT_DISTILLER_AGENT_ID} + generator_agent_id=${generatorAgentId} provenance=inferred lineage_class=derived derived_from=[entry.versionId] @@ -94,7 +95,7 @@ export function createResidentDistiller( description: opts.description ?? "Resident memory distiller — feed → classify → claim write", - systemPrompt: opts.systemPrompt ?? DEFAULT_SYSTEM_PROMPT, + systemPrompt: opts.systemPrompt ?? DEFAULT_SYSTEM_PROMPT(generatorAgentId), tools, capabilities: [...MEMORY_CAPABILITY_IDS], diff --git a/src/memory.ts b/src/memory.ts index 037dd7f..b7f5100 100644 --- a/src/memory.ts +++ b/src/memory.ts @@ -28,8 +28,10 @@ import { } from "./services/timeline.ts"; import { fetchFeed, + feedPageAfterAccessFilter, type FeedEntry, } from "./services/feed.ts"; + import { createTransformConfig, demoteGeneration, @@ -1388,9 +1390,9 @@ function createEngineDocumentStore(config: MemoryConfig): { createdAt: e.createdAt, accessTags: e.accessTags, })); - const nextCursor = - entries.length > 0 ? entries[entries.length - 1]!.feedSeq : null; - return { entries, nextCursor }; + // nextCursor must advance past the *raw* page even when ACL filters + // every entry — otherwise a denied page stalls the consumer forever. + return feedPageAfterAccessFilter(raw, entries); }, async close() { diff --git a/src/services/feed.test.ts b/src/services/feed.test.ts index 32cb8ed..c0b2c4e 100644 --- a/src/services/feed.test.ts +++ b/src/services/feed.test.ts @@ -4,6 +4,7 @@ import { FEED_LIMIT_MAX, FEED_LIMIT_MIN, FeedInputError, + feedPageAfterAccessFilter, } from "./feed.ts"; describe("feed constants", () => { @@ -21,3 +22,35 @@ describe("FeedInputError", () => { expect(err.message).toBe("bad cursor"); }); }); + +describe("feedPageAfterAccessFilter", () => { + it("keeps raw nextCursor when ACL denies the whole page", () => { + const raw = { + entries: [{ feedSeq: 10 }, { feedSeq: 11 }], + nextCursor: 11, + }; + const page = feedPageAfterAccessFilter(raw, []); + expect(page.entries).toEqual([]); + expect(page.nextCursor).toBe(11); + }); + + it("keeps raw nextCursor when some entries are allowed", () => { + const raw = { + entries: [{ feedSeq: 1 }, { feedSeq: 2 }, { feedSeq: 3 }], + nextCursor: 3, + }; + const allowed = [{ feedSeq: 1 }]; + const page = feedPageAfterAccessFilter(raw, allowed); + expect(page.entries).toEqual(allowed); + // Not the last allowed feedSeq (1) — advance past examined raw page. + expect(page.nextCursor).toBe(3); + }); + + it("returns null nextCursor when raw page is empty (end of feed)", () => { + const page = feedPageAfterAccessFilter( + { entries: [], nextCursor: null }, + [], + ); + expect(page.nextCursor).toBeNull(); + }); +}); diff --git a/src/services/feed.ts b/src/services/feed.ts index f56347e..f46ff2c 100644 --- a/src/services/feed.ts +++ b/src/services/feed.ts @@ -130,3 +130,20 @@ export async function fetchFeed( return { entries, nextCursor }; } + +/** + * Apply a document-access filter to a raw feed page. + * + * **Cursor rule:** `nextCursor` always comes from the raw examined page, + * not from the last allowed entry. A fully denied page must still advance + * the consumer past those `feed_seq` values or the poll stalls forever. + */ +export function feedPageAfterAccessFilter( + raw: { entries: readonly T[]; nextCursor: number | null }, + allowed: readonly T[], +): { entries: T[]; nextCursor: number | null } { + return { + entries: [...allowed], + nextCursor: raw.nextCursor, + }; +} From e9aa5dd924ec1e603d5d4fabfc0530b72e799989 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Thu, 13 Aug 2026 23:36:58 -0700 Subject: [PATCH 17/19] Fix stray knowledge references missed by the schema rename Comment references to memory.version.source_class in adapted-document.ts and a misleading example grant tag in grant-tags.test.ts still said knowledge; align with the memory rename. --- src/core/schemas/adapted-document.ts | 4 ++-- src/grant-tags.test.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/schemas/adapted-document.ts b/src/core/schemas/adapted-document.ts index 10642b4..1df3256 100644 --- a/src/core/schemas/adapted-document.ts +++ b/src/core/schemas/adapted-document.ts @@ -56,8 +56,8 @@ export type RawPointer = typeof RawPointerSchema.infer; // // Two orthogonal "class" axes on the write path: // - sourceClass: ranking prior (thread/channel/call/record/native) for -// computeAuthority — never written to knowledge.version.source_class. -// - lineageClass: data-lineage stored on knowledge.version.source_class +// computeAuthority — never written to memory.version.source_class. +// - lineageClass: data-lineage stored on memory.version.source_class // (native|imported|derived). Defaults to native at capture. // provenance: how the content was obtained (stated|inferred|unknown); // defaults to stated at capture for human/adapter paths. diff --git a/src/grant-tags.test.ts b/src/grant-tags.test.ts index d593655..5e2e4d9 100644 --- a/src/grant-tags.test.ts +++ b/src/grant-tags.test.ts @@ -43,13 +43,13 @@ describe("resolveAccessTags", () => { principalId: "u1", tenantId: "t1", accessTags: ["memory.space:eng"], - share: { tags: ["knowledge.project:ke"] }, + share: { tags: ["custom.project:ke"] }, }); expect(tags).toEqual( expect.arrayContaining([ ownerTag("u1"), "memory.space:eng", - "knowledge.project:ke", + "custom.project:ke", ]), ); }); From 2f93403c7335ee98dea8dc08bb05b01cf04c8240 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Thu, 13 Aug 2026 23:44:02 -0700 Subject: [PATCH 18/19] Make promote/demote generation swap atomic with embed activation (CL-5872) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolveActiveEmbedTable picked the tenant's newly-activated embed model independent of which generation's version rows were tagged live, so a promote or demote briefly left the active dense table pointed at one generation while `version.generation` still pointed at the other — live dense search silently returned zero rows for the gap between the two writes. Fold the embed-model activation and the generation-tag swap into a single Postgres transaction so a concurrent reader only ever observes the fully pre- or fully post-promote state. A thrown error mid-transaction now rolls back the embed activation along with the tag swap, so the manual restore-on-failure branch in promoteGeneration is no longer needed. createRawSqlClient now accepts a `sql.begin()` transaction handle in addition to the top-level connection, since embed-model-registry calls need to run inside the caller's transaction rather than committing on their own. --- src/core/embed-sql.ts | 11 ++- src/services/transform.ts | 166 +++++++++++++++----------------------- 2 files changed, 73 insertions(+), 104 deletions(-) diff --git a/src/core/embed-sql.ts b/src/core/embed-sql.ts index 76a87a3..10da901 100644 --- a/src/core/embed-sql.ts +++ b/src/core/embed-sql.ts @@ -1,4 +1,4 @@ -import type { RawSql } from "../db/client.ts"; +import type postgres from "postgres"; import type { EmbedRegistrySqlClient } from "./embed-model-registry.ts"; // Generic bridge from the engine's postgres-js handle to the minimal @@ -7,8 +7,13 @@ import type { EmbedRegistrySqlClient } from "./embed-model-registry.ts"; // Positional `$1`/`$2` placeholders via `sql.unsafe`. Despite the historical // name this is not embed-specific — `EmbedRegistrySqlClient` and // `FtsVerifySqlClient` (core/fts-language.ts) are structurally the same -// shape, so one bridge serves both. -export function createRawSqlClient(sql: RawSql): EmbedRegistrySqlClient { +// shape, so one bridge serves both. Accepts either the top-level connection +// or a `sql.begin()` transaction handle, so callers can fold registry writes +// into a larger transaction (see transform.ts promote/demote) instead of +// committing them separately. +export function createRawSqlClient( + sql: postgres.Sql<{}> | postgres.TransactionSql<{}>, +): EmbedRegistrySqlClient { return { async query(sqlText, params) { const rows = await sql.unsafe(sqlText, [...params] as never[]); diff --git a/src/services/transform.ts b/src/services/transform.ts index 5e33adf..af7b3c7 100644 --- a/src/services/transform.ts +++ b/src/services/transform.ts @@ -5,7 +5,6 @@ import type { EngineConfig } from "../config.ts"; import { newId } from "../core/id.ts"; import { formatCaughtError, log } from "../log.ts"; import { - memoryVersion, rawCapture, transformConfig, transformRun, @@ -554,68 +553,44 @@ export async function promoteGeneration( } const embed = buildEmbedClientConfig(configRow.params.embed, deps.config.embed); const archiveGen = `archive_${run.id}_${Date.now()}`; - const client = createRawSqlClient(deps.sql); + const readClient = createRawSqlClient(deps.sql); // Snapshot prior active model before we flip dense search. - const priorActive = await resolveActiveEmbedTable(client, input.tenantId); + const priorActive = await resolveActiveEmbedTable(readClient, input.tenantId); const priorModelKey = priorActive?.modelKey ?? null; - // Activate staged embed first — if this fails, versions stay put. - await activateEmbedModel(client, input.tenantId, embed); - - try { - await deps.db.transaction(async (tx) => { - // 1) archive current live - await tx - .update(memoryVersion) - .set({ generation: archiveGen }) - .where( - and( - eq(memoryVersion.tenantId, input.tenantId), - eq(memoryVersion.generation, LIVE_GENERATION), - ), - ); - // 2) promote staged → live - await tx - .update(memoryVersion) - .set({ generation: LIVE_GENERATION }) - .where( - and( - eq(memoryVersion.tenantId, input.tenantId), - eq(memoryVersion.generation, input.generation), - ), - ); - // 3) bookkeeping — generation column stays the original run id for lookup; - // versions now live under 'live'. Search by generation=runId after - // promote finds nothing (expected); demote restores. - await tx - .update(transformRun) - .set({ - archivedLiveGeneration: archiveGen, - archivedLiveModelKey: priorModelKey, - promotedAt: new Date(), - }) - .where(eq(transformRun.id, run.id)); - }); - } catch (err) { - // Version swap failed after dense activate — restore prior dense target. - try { - if (priorModelKey) { - await activateEmbedModelByKey(client, input.tenantId, priorModelKey); - } else { - await clearActiveEmbedModels(client, input.tenantId); - } - } catch (restoreErr) { - log.warn( - "promoteGeneration: failed to restore prior embed model after version swap error", - { - priorModelKey, - error: formatCaughtError(restoreErr), - }, - ); - } - throw err; - } + // Activate the staged embed model and swap generation tags inside one + // Postgres transaction. resolveActiveEmbedTable and the live-generation + // filter used by search are otherwise readable independently, which opens + // a window where dense search resolves the newly-active (staged) table + // while `version` rows are still tagged with the pre-swap generation — + // a silent, empty-result degradation of live dense search. Doing both + // writes in one transaction means any concurrent reader sees only the + // fully-pre-promote or fully-post-promote state, never the half-way one. + // A thrown error here rolls back the embed activation too, so no separate + // restore-on-failure step is needed. + await deps.sql.begin(async (txSql) => { + const txClient = createRawSqlClient(txSql); + await activateEmbedModel(txClient, input.tenantId, embed); + + // 1) archive current live + await txSql.unsafe( + `UPDATE "memory"."version" SET generation = $1 WHERE tenant_id = $2 AND generation = $3`, + [archiveGen, input.tenantId, LIVE_GENERATION], + ); + // 2) promote staged → live + await txSql.unsafe( + `UPDATE "memory"."version" SET generation = $1 WHERE tenant_id = $2 AND generation = $3`, + [LIVE_GENERATION, input.tenantId, input.generation], + ); + // 3) bookkeeping — generation column stays the original run id for lookup; + // versions now live under 'live'. Search by generation=runId after + // promote finds nothing (expected); demote restores. + await txSql.unsafe( + `UPDATE "memory"."transform_run" SET archived_live_generation = $1, archived_live_model_key = $2, promoted_at = now() WHERE id = $3`, + [archiveGen, priorModelKey, run.id], + ); + }); return loadTransformRun(deps.db, run.id); } @@ -653,52 +628,41 @@ export async function demoteGeneration( const archiveGen = run.archivedLiveGeneration; const priorModelKey = run.archivedLiveModelKey ?? null; - const client = createRawSqlClient(deps.sql); - - // Restore prior dense target first so a missing model_key fails closed - // before we rewrite generation tags. No prior model → clear active so - // dense degrades rather than remaining on the promoted table after swap. - if (priorModelKey) { - try { - await activateEmbedModelByKey(client, input.tenantId, priorModelKey); - } catch (err) { - throw new TransformPromoteError( - `cannot demote: failed to restore prior embed model ${priorModelKey}: ${formatCaughtError(err)}`, - ); + + // Restore prior dense target and rewrite generation tags inside one + // Postgres transaction — see promoteGeneration for why: doing these as + // separate writes lets a concurrent reader observe the restored model + // paired with not-yet-swapped generation tags (or vice versa), silently + // emptying live dense search for the duration of the gap. A thrown error + // here rolls back the restore too, so demote is all-or-nothing. + await deps.sql.begin(async (txSql) => { + const txClient = createRawSqlClient(txSql); + if (priorModelKey) { + try { + await activateEmbedModelByKey(txClient, input.tenantId, priorModelKey); + } catch (err) { + throw new TransformPromoteError( + `cannot demote: failed to restore prior embed model ${priorModelKey}: ${formatCaughtError(err)}`, + ); + } + } else { + await clearActiveEmbedModels(txClient, input.tenantId); } - } else { - await clearActiveEmbedModels(client, input.tenantId); - } - await deps.db.transaction(async (tx) => { // live (promoted) → back to run generation - await tx - .update(memoryVersion) - .set({ generation: input.generation }) - .where( - and( - eq(memoryVersion.tenantId, input.tenantId), - eq(memoryVersion.generation, LIVE_GENERATION), - ), - ); + await txSql.unsafe( + `UPDATE "memory"."version" SET generation = $1 WHERE tenant_id = $2 AND generation = $3`, + [input.generation, input.tenantId, LIVE_GENERATION], + ); // archive → live - await tx - .update(memoryVersion) - .set({ generation: LIVE_GENERATION }) - .where( - and( - eq(memoryVersion.tenantId, input.tenantId), - eq(memoryVersion.generation, archiveGen), - ), - ); - await tx - .update(transformRun) - .set({ - archivedLiveGeneration: null, - archivedLiveModelKey: null, - promotedAt: null, - }) - .where(eq(transformRun.id, run.id)); + await txSql.unsafe( + `UPDATE "memory"."version" SET generation = $1 WHERE tenant_id = $2 AND generation = $3`, + [LIVE_GENERATION, input.tenantId, archiveGen], + ); + await txSql.unsafe( + `UPDATE "memory"."transform_run" SET archived_live_generation = NULL, archived_live_model_key = NULL, promoted_at = NULL WHERE id = $1`, + [run.id], + ); }); return loadTransformRun(deps.db, run.id); From 5b6b04a55f857cbfcdcc22d75b3e82c7d4bf42a8 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Thu, 13 Aug 2026 23:44:09 -0700 Subject: [PATCH 19/19] Fix grantsMaterialized to reflect the access tag write, not just the grant (CL-5873) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit grantsMaterialized was set from isWritableGrantStore(grantStore) alone, so a host with a writable GrantStore but a DocumentStore that doesn't implement the optional appendAccessTags reported grantsMaterialized: true even though the memory.doc: tag was never stamped on the document — leaving the peer grant unreachable via canAccessDocument, which only checks tags actually present on the document. The flag now requires both the tag write and the grant write to have succeeded. Also wrap the tag-append and grant-materialize calls in try/catch: they run after store.add() has already durably committed the document, so a failure here must downgrade grantsMaterialized rather than reject add() for a document that in fact exists (which would invite a caller retry and a duplicate). --- src/memory.ts | 65 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/src/memory.ts b/src/memory.ts index b7f5100..6a9d3f9 100644 --- a/src/memory.ts +++ b/src/memory.ts @@ -8,7 +8,7 @@ import { } from "./grant-tags.ts"; import type { EngineConfig } from "./config.ts"; -import { log } from "./log.ts"; +import { formatCaughtError, log } from "./log.ts"; import { createDb, type Db, type RawSql } from "./db/client.ts"; import { createFtsVerification, parseFtsLanguage } from "./core/fts-language.ts"; import { createRawSqlClient } from "./core/embed-sql.ts"; @@ -869,34 +869,51 @@ function createPlaneFromStore( // Share materialization (CL-5873): stamp document-scoped tag + write // peer grants when the host store is writable. Tag mint alone is not - // enough for peers without host bootstrap grants on owner tags. + // enough for peers without host bootstrap grants on owner tags. The + // document is already durably committed by store.add() above, so a + // failure here must not throw (that would surface as add() failing + // for a document that in fact exists, inviting a caller retry that + // creates a duplicate) — it downgrades grantsMaterialized instead. + // grantsMaterialized is only ever true when BOTH the access tag was + // actually stamped AND the peer grant was actually written; either + // alone leaves canAccessDocument unable to find a match for peers. let grantsMaterialized: boolean | undefined; const peers = params.share?.principals; if (peers && peers.length > 0) { grantsMaterialized = false; - const docTag = documentTag(result.documentId); - if (store.appendAccessTags) { - await store.appendAccessTags(params.tenantId, result.documentId, [docTag]); - } else { - log.warn( - "memory.add: share.principals set but DocumentStore has no appendAccessTags; peer grants may not match", - { documentId: result.documentId }, - ); - } - if (isWritableGrantStore(grants?.grantStore)) { - await materializeShareGrants(grants.grantStore, { - tenantId: params.tenantId, - sharedByPrincipalId: params.principalId, - documentId: result.documentId, - sourceVersionId: result.versionId, - share: params.share ?? {}, - }); - grantsMaterialized = true; - } else { - log.warn( - "memory.add: share.principals set without WritableGrantStore; tags only (peers need host grants)", - { documentId: result.documentId }, + try { + const docTag = documentTag(result.documentId); + let tagStamped = false; + if (store.appendAccessTags) { + await store.appendAccessTags(params.tenantId, result.documentId, [docTag]); + tagStamped = true; + } else { + log.warn( + "memory.add: share.principals set but DocumentStore has no appendAccessTags; peer grants may not match", + { documentId: result.documentId }, + ); + } + if (isWritableGrantStore(grants?.grantStore)) { + await materializeShareGrants(grants.grantStore, { + tenantId: params.tenantId, + sharedByPrincipalId: params.principalId, + documentId: result.documentId, + sourceVersionId: result.versionId, + share: params.share ?? {}, + }); + grantsMaterialized = tagStamped; + } else { + log.warn( + "memory.add: share.principals set without WritableGrantStore; tags only (peers need host grants)", + { documentId: result.documentId }, + ); + } + } catch (err) { + log.error( + "memory.add: share materialization failed after document commit; peers may not have access", + { documentId: result.documentId, error: formatCaughtError(err) }, ); + grantsMaterialized = false; } }