diff --git a/CLAUDE.md b/CLAUDE.md index b2c7896..bc0d6d8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -275,7 +275,7 @@ See `docs/internal/ingest-architecture.md` for details. | `COPILOT_STORAGE_DIR` | auto-detected per OS | VS Code workspaceStorage root override | | `SMRITI_PROJECTS_ROOT` | `~/zero8.dev` | Projects root for ID derivation | | `OLLAMA_HOST` | `http://127.0.0.1:11434` | Ollama endpoint | -| `QMD_MEMORY_MODEL` | `qwen3:8b-tuned` | Ollama model for synthesis | +| `QMD_MEMORY_MODEL` | `qwen3.5:9b-mlx-tuned` | Ollama model for synthesis (MLX engine)| | `SMRITI_CLASSIFY_THRESHOLD` | `0.5` | LLM classification trigger threshold | | `SMRITI_AUTHOR` | `$USER` | Git author for team sharing | | `SMRITI_DAEMON_DEBOUNCE_MS` | `30000` | Daemon file-stability wait (v0.4.0) | diff --git a/package.json b/package.json index f76c6be..ef2b85c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "smriti", - "version": "0.8.2", + "version": "0.9.0", "description": "Smriti - Unified memory layer across all AI agents", "type": "module", "bin": { @@ -10,6 +10,8 @@ "dev": "bun --hot src/index.ts", "build": "bun build src/index.ts --outdir dist --target bun", "test": "bun test --cwd ./test", + "eval:recall": "bun run test/eval/recall-quality.eval.ts", + "eval:relations": "bun run test/eval/relation-inference.eval.ts", "smriti": "bun src/index.ts", "bench:qmd": "bun run scripts/bench-qmd.ts --profile ci-small --out bench/results/ci-small.json --no-llm", "bench:qmd:repeat": "bun run scripts/bench-qmd-repeat.ts --profiles ci-small,small,medium --runs 3 --out bench/results/repeat-summary.json", diff --git a/src/categorize/classifier.ts b/src/categorize/classifier.ts index 5bf17bd..347e9f5 100644 --- a/src/categorize/classifier.ts +++ b/src/categorize/classifier.ts @@ -7,7 +7,7 @@ import type { Database } from "bun:sqlite"; import { tagMessage, tagSession } from "../db"; -import { CLASSIFY_LLM_THRESHOLD, OLLAMA_HOST, OLLAMA_MODEL } from "../config"; +import { CLASSIFY_LLM_THRESHOLD, OLLAMA_HOST, requireOllamaModel } from "../config"; import { ALL_CATEGORY_IDS } from "./schema"; import { getRuleManager, type Rule } from "./rules/loader"; @@ -86,7 +86,7 @@ ${text.slice(0, 2000)}`; method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ - model: OLLAMA_MODEL, + model: requireOllamaModel(), prompt, stream: false, options: { temperature: 0.1, num_predict: 50 }, diff --git a/src/config.ts b/src/config.ts index 37c141f..2ce871b 100644 --- a/src/config.ts +++ b/src/config.ts @@ -68,7 +68,18 @@ export const PROJECTS_ROOT = // ============================================================================= export const OLLAMA_HOST = Bun.env.OLLAMA_HOST || "http://127.0.0.1:11434"; -export const OLLAMA_MODEL = Bun.env.QMD_MEMORY_MODEL || "qwen3:8b-tuned"; +export const OLLAMA_MODEL = Bun.env.QMD_MEMORY_MODEL; + +/** Resolve the Ollama model to use, preferring an explicit override. Throws if neither is set. */ +export function requireOllamaModel(explicit?: string): string { + const model = explicit || OLLAMA_MODEL; + if (!model) { + throw new Error( + "No Ollama model configured. Set QMD_MEMORY_MODEL in your environment or .env file." + ); + } + return model; +} /** Confidence threshold below which rule-based classification triggers LLM */ export const CLASSIFY_LLM_THRESHOLD = Number( diff --git a/src/db.ts b/src/db.ts index 59d8ba7..b9610a4 100644 --- a/src/db.ts +++ b/src/db.ts @@ -8,12 +8,13 @@ */ import { Database } from "bun:sqlite"; -import { mkdirSync, existsSync } from "fs"; -import { dirname } from "path"; -import { QMD_DB_PATH, SMRITI_SESSIONS_DIR } from "./config"; -import { initializeMemoryTables } from "./qmd"; +import { mkdirSync, existsSync, unlinkSync } from "fs"; +import { dirname, join } from "path"; +import { QMD_DB_PATH, SMRITI_SESSIONS_DIR, SMRITI_DIR } from "./config"; +import { initializeMemoryTables, deleteSession, cleanupOrphanedMemoryVectors } from "./qmd"; import { createStore } from "../qmd/src/index"; import { setQmdStore, closeQmdStore } from "./store"; +import type { KnowledgeUnit } from "./team/types"; // ============================================================================= // Connection @@ -204,6 +205,67 @@ export function initializeSmritiTables(db: Database): void { entities TEXT ); + -- Knowledge consolidation: raw Stage-1 extracts, promoted to canonical on reuse + CREATE TABLE IF NOT EXISTS smriti_knowledge_units ( + id TEXT PRIMARY KEY, -- KnowledgeUnit.id (uuid) + session_id TEXT NOT NULL, + project_id TEXT, + topic TEXT NOT NULL, + category TEXT NOT NULL, + relevance REAL NOT NULL DEFAULT 0, -- 0-10, from Stage 1 + entities TEXT, -- JSON array + files TEXT, -- JSON array + plain_text TEXT NOT NULL, -- raw Stage-1 extract + line_ranges TEXT, -- JSON array of {start,end} + content_hash TEXT NOT NULL, -- hashContent({topic,category,plainText}) — Stage-1 dedup key + tier TEXT NOT NULL DEFAULT 'segmented', -- 'segmented' | 'canonical' | 'archived' + retrieval_count INTEGER NOT NULL DEFAULT 0, + last_recalled_at TEXT, + promoted_at TEXT, + canonical_doc_path TEXT, -- relative path under .smriti/knowledge/, set on promotion + share_id TEXT, -- points at the smriti_shares row created on promotion + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + + CREATE INDEX IF NOT EXISTS idx_smriti_knowledge_units_session ON smriti_knowledge_units(session_id); + CREATE INDEX IF NOT EXISTS idx_smriti_knowledge_units_hash ON smriti_knowledge_units(content_hash); + CREATE INDEX IF NOT EXISTS idx_smriti_knowledge_units_tier ON smriti_knowledge_units(tier); + + -- Canonical entity registry: resolves free-text entity mentions (from Stage 1 + -- extraction) onto a stable node, so recurrence is detected regardless of wording. + -- Propagated team/org-wide via .smriti/config.json, same mechanism as custom categories. + CREATE TABLE IF NOT EXISTS smriti_entities ( + id TEXT PRIMARY KEY, -- slug, e.g. "jwt", "redis" + label TEXT NOT NULL, -- canonical display name + entity_type TEXT NOT NULL DEFAULT 'concept', -- 'technology' | 'concept' | 'file' | 'pattern' + aliases TEXT NOT NULL DEFAULT '[]', -- JSON array of raw strings seen + mention_count INTEGER NOT NULL DEFAULT 0, + first_seen_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE INDEX IF NOT EXISTS idx_smriti_entities_label ON smriti_entities(label); + + -- Relationship triples (subject/object polymorphic via type+id, not literal RDF URIs). + -- knowledge_unit -mentions-> entity edges come free from Stage-1 extraction; + -- knowledge_unit -relatesTo/supersedes/contradicts-> knowledge_unit edges are LLM-gated, + -- only at promotion time (see src/learn/consolidate.ts), persisting what + -- ollamaCheckConflicts previously only computed ephemerally. + CREATE TABLE IF NOT EXISTS smriti_relationships ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + subject_type TEXT NOT NULL, -- 'knowledge_unit' | 'entity' | 'session' + subject_id TEXT NOT NULL, + predicate TEXT NOT NULL, -- 'mentions' | 'relatesTo' | 'supersedes' | 'contradicts' + object_type TEXT NOT NULL, + object_id TEXT NOT NULL, + confidence REAL DEFAULT 1.0, + source TEXT DEFAULT 'extraction', -- 'extraction' | 'derived' | 'llm' + created_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE(subject_type, subject_id, predicate, object_type, object_id) + ); + CREATE INDEX IF NOT EXISTS idx_smriti_relationships_subject ON smriti_relationships(subject_type, subject_id); + CREATE INDEX IF NOT EXISTS idx_smriti_relationships_object ON smriti_relationships(object_type, object_id); + CREATE INDEX IF NOT EXISTS idx_smriti_relationships_predicate ON smriti_relationships(predicate); + -- Tool usage tracking CREATE TABLE IF NOT EXISTS smriti_tool_usage ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -425,6 +487,21 @@ export function initializeSmritiTables(db: Database): void { DELETE FROM smriti_queries_fts WHERE rowid = old.id; END; `); + + // Prune: 'archived' tier support on smriti_knowledge_units (no CHECK + // constraint on `tier`, so the new value needs no migration — only these + // two nullable columns, set when a canonical unit is archived because a + // `supersedes` edge points at it). + try { + db.exec(`ALTER TABLE smriti_knowledge_units ADD COLUMN archived_at TEXT`); + } catch { + // Column already exists + } + try { + db.exec(`ALTER TABLE smriti_knowledge_units ADD COLUMN archived_reason TEXT`); + } catch { + // Column already exists + } } // ============================================================================= @@ -475,6 +552,12 @@ const DEFAULT_AGENTS = [ log_pattern: null, parser: "claude-web", }, + { + id: "team", + display_name: "Team Import", + log_pattern: null, + parser: "generic", + }, ] as const; /** Default category taxonomy */ @@ -1092,6 +1175,31 @@ export function deleteSidecarRows(db: Database, sessionId: string): void { db.prepare(`DELETE FROM smriti_session_costs WHERE session_id = ?`).run(sessionId); } +/** + * Full sidecar cleanup for a session forget — a superset of deleteSidecarRows + * (which `ingest --force` uses, needing only the narrower tool/file/command/ + * error/cost set that gets re-derived on re-ingest). Also clears + * Smriti-specific metadata/content tables that didn't exist when + * deleteSidecarRows was written. Does NOT touch smriti_knowledge_units or + * smriti_shares — callers (forgetSession) handle those separately since + * canonical (promoted) units are kept unless purging shared knowledge. + */ +export function deleteAllSidecarRows(db: Database, sessionId: string): void { + deleteSidecarRows(db, sessionId); + + db.prepare( + `DELETE FROM smriti_message_tags WHERE message_id IN (SELECT id FROM memory_messages WHERE session_id = ?)` + ).run(sessionId); + db.prepare(`DELETE FROM smriti_session_meta WHERE session_id = ?`).run(sessionId); + db.prepare(`DELETE FROM smriti_session_tags WHERE session_id = ?`).run(sessionId); + db.prepare(`DELETE FROM smriti_artifacts WHERE session_id = ?`).run(sessionId); + db.prepare(`DELETE FROM smriti_thinking WHERE session_id = ?`).run(sessionId); + db.prepare(`DELETE FROM smriti_attachments WHERE session_id = ?`).run(sessionId); + db.prepare(`DELETE FROM smriti_voice_notes WHERE session_id = ?`).run(sessionId); + db.prepare(`DELETE FROM smriti_session_queries WHERE session_id = ?`).run(sessionId); + db.prepare(`DELETE FROM smriti_session_clusters WHERE session_id = ?`).run(sessionId); +} + export function insertGitOperation( db: Database, messageId: number, @@ -1246,6 +1354,366 @@ export function getDensityScore(db: Database, sessionId: string): number { return row?.density_score ?? 0; } +// ============================================================================= +// Knowledge Consolidation (Progressive Summarization) +// ============================================================================= + +export interface StoredKnowledgeUnit { + id: string; + session_id: string; + project_id: string | null; + topic: string; + category: string; + relevance: number; + entities: string[]; + files: string[]; + plain_text: string; + line_ranges: Array<{ start: number; end: number }>; + content_hash: string; + tier: "segmented" | "canonical" | "archived"; + retrieval_count: number; + last_recalled_at: string | null; + promoted_at: string | null; + canonical_doc_path: string | null; + share_id: string | null; + archived_at: string | null; + archived_reason: string | null; +} + +type KnowledgeUnitRow = { + id: string; + session_id: string; + project_id: string | null; + topic: string; + category: string; + relevance: number; + entities: string | null; + files: string | null; + plain_text: string; + line_ranges: string | null; + content_hash: string; + tier: string; + retrieval_count: number; + last_recalled_at: string | null; + promoted_at: string | null; + canonical_doc_path: string | null; + share_id: string | null; + archived_at: string | null; + archived_reason: string | null; +}; + +function deserializeKnowledgeUnit(row: KnowledgeUnitRow): StoredKnowledgeUnit { + return { + ...row, + entities: row.entities ? JSON.parse(row.entities) : [], + files: row.files ? JSON.parse(row.files) : [], + line_ranges: row.line_ranges ? JSON.parse(row.line_ranges) : [], + tier: row.tier as "segmented" | "canonical" | "archived", + }; +} + +/** + * Insert a Stage-1 knowledge unit if its content hash isn't already stored. + * Returns true if inserted, false if it was a duplicate (caller distinguishes + * "stored" from "skipped" the same way shareSegmentedKnowledge does for shares). + */ +export function insertKnowledgeUnit( + db: Database, + unit: KnowledgeUnit, + sessionId: string, + projectId: string | null, + contentHash: string +): boolean { + const exists = db + .prepare(`SELECT 1 FROM smriti_knowledge_units WHERE content_hash = ?`) + .get(contentHash); + if (exists) return false; + + db.prepare( + `INSERT INTO smriti_knowledge_units + (id, session_id, project_id, topic, category, relevance, entities, files, plain_text, line_ranges, content_hash) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + unit.id, + sessionId, + projectId, + unit.topic, + unit.category, + unit.relevance, + JSON.stringify(unit.entities || []), + JSON.stringify(unit.files || []), + unit.plainText, + JSON.stringify(unit.lineRanges || []), + contentHash + ); + return true; +} + +/** Dense sessions (by density_score) that haven't been segmented into knowledge units yet. */ +export function findUnsegmentedDenseSessions( + db: Database, + minDensity: number, + limit?: number +): Array<{ session_id: string; project_id: string | null; density_score: number }> { + const query = ` + SELECT sm.session_id, sm.project_id, sm.density_score + FROM smriti_session_meta sm + WHERE sm.density_score >= ? + AND NOT EXISTS (SELECT 1 FROM smriti_knowledge_units ku WHERE ku.session_id = sm.session_id) + ORDER BY sm.density_score DESC + ${limit ? "LIMIT ?" : ""} + `; + const rows = limit + ? db.prepare(query).all(minDensity, limit) + : db.prepare(query).all(minDensity); + return rows as Array<{ session_id: string; project_id: string | null; density_score: number }>; +} + +/** Segmented units that have proven reuse (via recall) or scored high relevance at extraction time. */ +export function findPromotableUnits( + db: Database, + minRetrievals: number, + minRelevance: number, + minEntityReach?: number +): StoredKnowledgeUnit[] { + // minEntityReach: a unit is promotable if one of its entities is + // independently mentioned by >= minEntityReach OTHER units — a structural + // reuse signal (cross-session recurrence) that doesn't depend on recall() + // ever having been called on this particular unit. + const entityReachClause = minEntityReach + ? `OR id IN ( + SELECT r1.subject_id FROM smriti_relationships r1 + JOIN smriti_relationships r2 + ON r1.object_id = r2.object_id AND r2.predicate = 'mentions' + AND r1.predicate = 'mentions' AND r1.subject_id != r2.subject_id + WHERE r1.subject_type = 'knowledge_unit' + GROUP BY r1.subject_id + HAVING COUNT(DISTINCT r2.subject_id) >= ? + )` + : ""; + const params = minEntityReach + ? [minRetrievals, minRelevance, minEntityReach] + : [minRetrievals, minRelevance]; + + const rows = db + .prepare( + `SELECT * FROM smriti_knowledge_units + WHERE tier = 'segmented' AND (retrieval_count >= ? OR relevance >= ? ${entityReachClause})` + ) + .all(...params) as KnowledgeUnitRow[]; + return rows.map(deserializeKnowledgeUnit); +} + +/** Bump retrieval_count for any knowledge units belonging to a recalled session. No-op if none exist yet. */ +export function incrementRetrievalCount(db: Database, sessionId: string): void { + db.prepare( + `UPDATE smriti_knowledge_units + SET retrieval_count = retrieval_count + 1, + last_recalled_at = datetime('now'), + updated_at = datetime('now') + WHERE session_id = ?` + ).run(sessionId); +} + +export function promoteKnowledgeUnit( + db: Database, + unitId: string, + canonicalDocPath: string, + shareId: string +): void { + db.prepare( + `UPDATE smriti_knowledge_units + SET tier = 'canonical', promoted_at = datetime('now'), + canonical_doc_path = ?, share_id = ?, updated_at = datetime('now') + WHERE id = ?` + ).run(canonicalDocPath, shareId, unitId); +} + +export function listKnowledgeUnits( + db: Database, + options: { tier?: "segmented" | "canonical" | "archived"; minRetrievals?: number; limit?: number } = {} +): StoredKnowledgeUnit[] { + const conditions: string[] = []; + const params: any[] = []; + + if (options.tier) { + conditions.push("tier = ?"); + params.push(options.tier); + } + if (options.minRetrievals !== undefined) { + conditions.push("retrieval_count >= ?"); + params.push(options.minRetrievals); + } + + const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : ""; + const limitClause = options.limit ? "LIMIT ?" : ""; + if (options.limit) params.push(options.limit); + + const rows = db + .prepare( + `SELECT * FROM smriti_knowledge_units ${where} + ORDER BY retrieval_count DESC, relevance DESC + ${limitClause}` + ) + .all(...params) as KnowledgeUnitRow[]; + return rows.map(deserializeKnowledgeUnit); +} + +/** Cascade-delete relationship edges where this knowledge unit is subject or object. */ +function deleteKnowledgeUnitRelationships(db: Database, unitId: string): void { + db.prepare( + `DELETE FROM smriti_relationships WHERE subject_type = 'knowledge_unit' AND subject_id = ?` + ).run(unitId); + db.prepare( + `DELETE FROM smriti_relationships WHERE object_type = 'knowledge_unit' AND object_id = ?` + ).run(unitId); +} + +/** + * Hard-delete a knowledge unit and its relationship edges. Shared by + * forgetSession (removing unpromoted units of a forgotten session) and + * pruneKnowledge (removing stale segmented units) — safe in both cases + * because a 'segmented' unit was never promoted, so nothing external + * (canonical doc, smriti_shares row) references it. + */ +export function deleteKnowledgeUnit(db: Database, unitId: string): void { + deleteKnowledgeUnitRelationships(db, unitId); + db.prepare(`DELETE FROM smriti_knowledge_units WHERE id = ?`).run(unitId); +} + +/** + * Segmented units that failed both promotion paths — the relevance escape + * hatch mirrors findPromotableUnits' own minRelevance, so a unit one + * `consolidate` run away from promoting is never a prune candidate — and are + * old enough that they're unlikely to ever clear the bar. + */ +export function findStaleSegmentedUnits( + db: Database, + maxAgeDays: number, + minRelevance: number +): StoredKnowledgeUnit[] { + const rows = db + .prepare( + `SELECT * FROM smriti_knowledge_units + WHERE tier = 'segmented' AND retrieval_count = 0 AND relevance < ? + AND created_at < datetime('now', '-' || ? || ' days')` + ) + .all(minRelevance, maxAgeDays) as KnowledgeUnitRow[]; + return rows.map(deserializeKnowledgeUnit); +} + +/** Canonical units with an incoming `supersedes` edge (some other unit supersedes them) that aren't already archived. */ +export function findSupersededCanonicalUnits( + db: Database +): Array { + const rows = db + .prepare( + `SELECT ku.*, r.subject_id AS supersededByUnitId, super_ku.topic AS supersededByTopic + FROM smriti_knowledge_units ku + JOIN smriti_relationships r + ON r.object_type = 'knowledge_unit' AND r.object_id = ku.id AND r.predicate = 'supersedes' + JOIN smriti_knowledge_units super_ku ON super_ku.id = r.subject_id + WHERE ku.tier = 'canonical'` + ) + .all() as Array; + return rows.map((r) => ({ ...deserializeKnowledgeUnit(r), supersededByUnitId: r.supersededByUnitId, supersededByTopic: r.supersededByTopic })); +} + +/** Soft-archive a canonical unit — tier -> 'archived', archived_at/reason set. The unit's relationship edges (including the supersedes edge that justified this) are left untouched as the audit trail. */ +export function archiveKnowledgeUnit(db: Database, unitId: string, reason: string): void { + db.prepare( + `UPDATE smriti_knowledge_units + SET tier = 'archived', archived_at = datetime('now'), archived_reason = ?, updated_at = datetime('now') + WHERE id = ?` + ).run(reason, unitId); +} + +// ============================================================================= +// Forget (session deletion) +// ============================================================================= + +export type ForgetOptions = { + /** Permanently delete instead of the default soft delete (active = 0). */ + hard?: boolean; + /** Only meaningful with hard: true. Also delete canonical (promoted) units, their smriti_shares row, and their .smriti/knowledge/*.md doc — normally kept since they've already been shared. */ + purgeShared?: boolean; + /** Where canonical docs live, for purgeShared's file deletion. Defaults to the same convention consolidateKnowledge uses. */ + outputDir?: string; +}; + +export type ForgetResult = { + sessionId: string; + hard: boolean; + unitsDeleted: number; // unpromoted (segmented) knowledge units removed + unitsPurged: number; // canonical units removed, only when purgeShared + canonicalKept: number; // canonical units left in place +}; + +/** + * Forget a session. Soft delete (default) just flips memory_sessions.active + * to 0 — reversible, and already understood by `list --all`/`listSessions`. + * Hard delete removes messages, all sidecar rows, unpromoted knowledge + * units, and orphaned vector embeddings; canonical (promoted) units are kept + * unless purgeShared is set, since they may already be referenced outside + * this session (team sync, a committed .smriti/knowledge/ doc). + */ +export function forgetSession( + db: Database, + sessionId: string, + options: ForgetOptions = {} +): ForgetResult { + const hard = options.hard ?? false; + const purgeShared = options.purgeShared ?? false; + const result: ForgetResult = { + sessionId, + hard, + unitsDeleted: 0, + unitsPurged: 0, + canonicalKept: 0, + }; + + if (!hard) { + deleteSession(db as any, sessionId, false); + return result; + } + + const units = db + .prepare( + `SELECT id, tier, canonical_doc_path FROM smriti_knowledge_units WHERE session_id = ?` + ) + .all(sessionId) as Array<{ id: string; tier: string; canonical_doc_path: string | null }>; + + const outputDir = options.outputDir || join(process.cwd(), SMRITI_DIR); + + for (const u of units) { + if (u.tier !== "canonical") { + deleteKnowledgeUnit(db, u.id); + result.unitsDeleted++; + continue; + } + if (!purgeShared) { + result.canonicalKept++; + continue; + } + deleteKnowledgeUnit(db, u.id); + db.prepare(`DELETE FROM smriti_shares WHERE unit_id = ?`).run(u.id); + if (u.canonical_doc_path) { + try { + unlinkSync(join(outputDir, u.canonical_doc_path)); + } catch { + // Doc already gone or never written under this outputDir — fine. + } + } + result.unitsPurged++; + } + + deleteAllSidecarRows(db, sessionId); + deleteSession(db as any, sessionId, true); + cleanupOrphanedMemoryVectors(db as any); + + return result; +} + // ============================================================================= // Session Query Labels (#60) // ============================================================================= diff --git a/src/format.ts b/src/format.ts index 023b0c3..b32812f 100644 --- a/src/format.ts +++ b/src/format.ts @@ -268,6 +268,123 @@ export function formatShareResult(result: { return lines.join("\n"); } +// ============================================================================= +// Consolidate Result Formatting +// ============================================================================= + +export function formatConsolidateResult(result: { + sessionsSegmented: number; + unitsStored: number; + unitsSkipped: number; + unitsPromoted: number; + unitsPruned?: number; + unitsArchived?: number; + pruneCandidates?: Array<{ id: string; topic: string; tier: string; action: string; reason: string }>; + errors: string[]; +}): string { + const lines = [ + `Sessions segmented: ${result.sessionsSegmented}`, + `Units stored: ${result.unitsStored}`, + `Units skipped (dedup): ${result.unitsSkipped}`, + `Units promoted: ${result.unitsPromoted}`, + ]; + + if (result.pruneCandidates && result.pruneCandidates.length > 0) { + lines.push(""); + lines.push(`Prune candidates (dry-run — rerun with --yes to apply):`); + lines.push( + table( + ["Topic", "Tier", "Action", "Reason"], + result.pruneCandidates.map((c) => [c.topic, c.tier, c.action, c.reason]) + ) + ); + } else if (result.unitsPruned !== undefined || result.unitsArchived !== undefined) { + lines.push(`Units pruned (deleted): ${result.unitsPruned ?? 0}`); + lines.push(`Units archived (superseded): ${result.unitsArchived ?? 0}`); + } + + if (result.errors.length > 0) { + lines.push(`Errors: ${result.errors.length}`); + for (const err of result.errors.slice(0, 5)) { + lines.push(` - ${err}`); + } + } + + return lines.join("\n"); +} + +// ============================================================================= +// Knowledge Units (Learnings) Formatting +// ============================================================================= + +export function formatLearnings( + units: Array<{ + tier: string; + topic: string; + category: string; + retrieval_count: number; + relevance: number; + canonical_doc_path: string | null; + }> +): string { + if (units.length === 0) return "No knowledge units found."; + + const headers = ["Tier", "Topic", "Category", "Retrievals", "Relevance", "Doc Path"]; + const rows = units.map((u) => [ + u.tier === "canonical" ? "✓ canonical" : "segmented", + u.topic, + u.category, + String(u.retrieval_count), + u.relevance.toFixed(1), + u.canonical_doc_path || "-", + ]); + + return table(headers, rows, [14, 40, 20, 10, 9, 40]); +} + +// ============================================================================= +// Entity Graph Formatting (smriti graph ) +// ============================================================================= + +export function formatEntityGraph( + entity: { id: string; label: string; entity_type: string; aliases: string[]; mention_count: number }, + units: Array<{ id: string; topic: string; category: string; relevance: number; tier: string; retrieval_count: number }>, + edges: Array<{ subject_id: string; predicate: string; object_id: string }> +): string { + const lines = [ + `Entity: ${entity.label} (${entity.id})`, + `Type: ${entity.entity_type}`, + `Aliases: ${entity.aliases.join(", ") || "-"}`, + `Mentioned ${entity.mention_count} time(s) across ${units.length} unit(s)`, + "", + ]; + + if (units.length === 0) { + lines.push("No knowledge units mention this entity yet."); + return lines.join("\n"); + } + + const headers = ["Tier", "Topic", "Category", "Retrievals", "Relevance"]; + const rows = units.map((u) => [ + u.tier === "canonical" ? "✓ canonical" : "segmented", + u.topic, + u.category, + String(u.retrieval_count), + u.relevance.toFixed(1), + ]); + lines.push(table(headers, rows, [14, 40, 20, 10, 9])); + + const nonMentionEdges = edges.filter((e) => e.predicate !== "mentions"); + if (nonMentionEdges.length > 0) { + lines.push("", "Relationships between these units:"); + for (const e of nonMentionEdges) { + lines.push(` ${e.subject_id} --${e.predicate}--> ${e.object_id}`); + } + } + + return lines.join("\n"); +} + // ============================================================================= // Sync Result Formatting // ============================================================================= @@ -278,6 +395,7 @@ export function formatSyncResult(result: { skipped: number; errors: string[]; categoriesImported?: number; + entitiesImported?: number; }): string { const lines = [ `Files processed: ${result.filesProcessed}`, @@ -287,6 +405,9 @@ export function formatSyncResult(result: { if (result.categoriesImported && result.categoriesImported > 0) { lines.push(`Categories imported: ${result.categoriesImported}`); } + if (result.entitiesImported && result.entitiesImported > 0) { + lines.push(`Entities imported: ${result.entitiesImported}`); + } if (result.errors.length > 0) { lines.push(`Errors: ${result.errors.length}`); diff --git a/src/index.ts b/src/index.ts index bb186d5..ab6a2e7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,7 +7,7 @@ * schema-based categorization, and team knowledge sharing. */ -import { initSmriti, closeDb, getCategories, getCategoryTree, addCategory, listProjects, tagSession, getProjectReport, getTagUsage, computeDensityScore, updateDensityScore, insertSessionQueries, getUnenrichedSessionIds } from "./db"; +import { initSmriti, closeDb, getCategories, getCategoryTree, addCategory, listProjects, tagSession, getProjectReport, type ProjectInspectReport, getTagUsage, type TagUsageEntry, computeDensityScore, updateDensityScore, insertSessionQueries, getUnenrichedSessionIds, listKnowledgeUnits, forgetSession } from "./db"; import { getMessages, getSession, getMemoryStatus, embedMemoryMessages } from "./qmd"; import { ingest, ingestAll } from "./ingest/index"; import { categorizeUncategorized } from "./categorize/classifier"; @@ -16,6 +16,8 @@ import { searchFiltered, listSessions } from "./search/index"; import { recall } from "./search/recall"; import { shareKnowledge } from "./team/share"; import { syncTeamKnowledge, listTeamContributions } from "./team/sync"; +import { consolidateKnowledge } from "./learn/consolidate"; +import { findEntity, getUnitsForEntity, getRelationships } from "./learn/entities"; import { generateContext, compareSessions, @@ -53,6 +55,9 @@ import { formatTagUsage, formatDensityBreakdown, formatDigest, + formatConsolidateResult, + formatLearnings, + formatEntityGraph, json, } from "./format"; import { generateDigest } from "./digest"; @@ -206,6 +211,8 @@ Commands: recall [options] Smart recall with optional synthesis categorize [options] Auto-categorize sessions tag Manually tag a session + forget [opts] Delete a session (soft by default; --hard --yes for real deletion) + forget --all [filters] Bulk forget, reusing list's --project/--category/--agent filters categories List category tree categories add [opts] Add a custom category tags [options] Show tag usage in sessions @@ -213,6 +220,9 @@ Commands: compare Compare two sessions (tokens, tools, files) compare --last Compare last 2 sessions for current project share [filters] Export knowledge to .smriti/ + consolidate [options] Segment dense sessions, promote reused units, prune stale/superseded ones + learnings [options] List extracted knowledge units (tier, retrievals, relevance) + graph Show a canonical entity's mentions and relationship edges sync Import team knowledge from .smriti/ team View team contributions list [filters] List sessions @@ -238,6 +248,12 @@ Filters (apply to search, recall, list, share): --agent Filter by agent --limit Max results (default varies by command) +Forget options: + --hard Permanently delete instead of soft delete (requires --yes) + --yes Confirm --hard (required — no confirmation prompt otherwise) + --purge-shared With --hard, also delete canonical (promoted) units, their + smriti_shares row, and their .smriti/knowledge/*.md doc + Ingest options: smriti ingest claude Ingest Claude Code sessions smriti ingest claude-web Claude.ai data export @@ -280,6 +296,11 @@ Share options: --segmented Use 3-stage segmentation pipeline (beta) --min-relevance Relevance threshold for segmented mode (default: 6) +Consolidate options: + --prune Also run the prune phase (dry-run by default — prints candidates, deletes nothing) + --yes, --apply Actually delete/archive prune candidates (requires --prune) + --prune-stale-days Age threshold for stale segmented units (default: 30) + Insights options: smriti insights Full dashboard smriti insights session Session deep dive @@ -302,6 +323,9 @@ Examples: smriti search "auth" --project myapp smriti recall "how did we set up auth" --synthesize smriti categorize + smriti consolidate + smriti consolidate --prune + smriti consolidate --prune --yes smriti list --category decision --project myapp smriti share --category decision smriti sync @@ -642,6 +666,61 @@ async function main() { break; } + // ===================================================================== + // FORGET + // ===================================================================== + case "forget": { + const all = hasFlag(args, "--all"); + const sessionId = getPositional(args, 1); + if (!sessionId && !all) { + console.error("Usage: smriti forget [--hard] [--yes] [--purge-shared]"); + console.error(" smriti forget --all [--project ] [--category ] [--agent ] [--hard] [--yes] [--purge-shared]"); + process.exit(1); + } + + const hard = hasFlag(args, "--hard"); + const purgeShared = hasFlag(args, "--purge-shared"); + if (hard && !hasFlag(args, "--yes")) { + console.error("--hard permanently deletes session data. Re-run with --yes to confirm."); + process.exit(1); + } + + const targetIds = all + ? listSessions(db, { + project: getArg(args, "--project"), + category: getArg(args, "--category"), + agent: getArg(args, "--agent"), + includeInactive: true, + }).map((s) => s.id) + : [sessionId!]; + + if (targetIds.length === 0) { + console.log("No matching sessions to forget."); + break; + } + + let deleted = 0; + let purged = 0; + let kept = 0; + for (const id of targetIds) { + const r = forgetSession(db, id, { hard, purgeShared }); + deleted += r.unitsDeleted; + purged += r.unitsPurged; + kept += r.canonicalKept; + } + + console.log(`Forgot ${targetIds.length} session(s) (${hard ? "hard delete" : "soft delete"}).`); + if (hard) { + console.log(` Unpromoted knowledge units removed: ${deleted}`); + if (purgeShared) { + console.log(` Canonical knowledge units purged: ${purged}`); + } else if (kept > 0) { + console.log(` Canonical knowledge units kept (already shared — pass --purge-shared to also remove): ${kept}`); + } + } + break; + } + // ===================================================================== // CATEGORIES // ===================================================================== @@ -812,6 +891,82 @@ async function main() { break; } + // ===================================================================== + // CONSOLIDATE + // ===================================================================== + case "consolidate": { + const prune = hasFlag(args, "--prune"); + const pruneApply = hasFlag(args, "--yes") || hasFlag(args, "--apply"); + const result = await consolidateKnowledge(db, { + minDensity: Number(getArg(args, "--min-density")) || undefined, + minRetrievals: Number(getArg(args, "--min-retrievals")) || undefined, + minRelevance: Number(getArg(args, "--min-relevance")) || undefined, + minEntityReach: Number(getArg(args, "--min-entity-reach")) || undefined, + model: getArg(args, "--model"), + outputDir: getArg(args, "--output"), + sessionLimit: Number(getArg(args, "--session-limit")) || undefined, + prune, + pruneStaleDays: Number(getArg(args, "--prune-stale-days")) || undefined, + pruneApply, + onProgress: (msg) => console.log(` ${msg}`), + }); + + console.log(formatConsolidateResult(result)); + if (prune && !pruneApply && result.pruneCandidates && result.pruneCandidates.length > 0) { + console.log("\nRun again with --prune --yes to apply."); + } + break; + } + + // ===================================================================== + // LEARNINGS + // ===================================================================== + case "learnings": { + const units = listKnowledgeUnits(db, { + tier: getArg(args, "--tier") as "segmented" | "canonical" | undefined, + minRetrievals: Number(getArg(args, "--min-retrievals")) || undefined, + limit: Number(getArg(args, "--limit")) || 50, + }); + + if (hasFlag(args, "--json")) { + console.log(json(units)); + } else { + console.log(formatLearnings(units)); + } + break; + } + + // ===================================================================== + // GRAPH + // ===================================================================== + case "graph": { + const query = getPositional(args, 1); + if (!query) { + console.error("Usage: smriti graph "); + process.exit(1); + } + + const entity = findEntity(db, query); + if (!entity) { + console.log(`No entity found matching "${query}".`); + break; + } + + const units = getUnitsForEntity(db, entity.id); + const unitIds = new Set(units.map((u) => u.id)); + const edges = units.flatMap((u) => + getRelationships(db, { subjectType: "knowledge_unit", subjectId: u.id }) + .filter((r) => r.predicate !== "mentions" && unitIds.has(r.object_id)) + ); + + if (hasFlag(args, "--json")) { + console.log(json({ entity, units, edges })); + } else { + console.log(formatEntityGraph(entity, units, edges)); + } + break; + } + // ===================================================================== // SYNC // ===================================================================== diff --git a/src/learn/consolidate.ts b/src/learn/consolidate.ts new file mode 100644 index 0000000..c57326f --- /dev/null +++ b/src/learn/consolidate.ts @@ -0,0 +1,612 @@ +/** + * learn/consolidate.ts - Continuous knowledge consolidation + * + * Progressive Summarization: cheap Stage-1 extraction runs broadly over dense + * sessions; expensive Stage-2 polish only runs once a unit proves it's reused + * (recalled repeatedly) or scored high relevance at extraction time. + * + * Two independent phases, run sequentially: + * - Segment: dense, not-yet-segmented sessions -> segmentSession() -> smriti_knowledge_units + * - Promote: knowledge units that cleared the reuse/relevance bar -> generateDocument() + * -> written to .smriti/knowledge/ + recorded in smriti_shares + * + * CLI-only, like `categorize`/`share` — never wired into the daemon (see + * src/daemon/index.ts's enrichOnIngest comment for why LLM work per-flush is unsafe). + */ + +import type { Database } from "bun:sqlite"; +import { mkdirSync } from "fs"; +import { join } from "path"; +import { SMRITI_DIR, AUTHOR } from "../config"; +import { hashContent } from "../qmd"; +import { + findUnsegmentedDenseSessions, + insertKnowledgeUnit, + findPromotableUnits, + promoteKnowledgeUnit, + findStaleSegmentedUnits, + findSupersededCanonicalUnits, + deleteKnowledgeUnit, + archiveKnowledgeUnit, +} from "../db"; +import { getSessionMessages } from "../team/share"; +import { segmentSession } from "../team/segment"; +import { generateDocument, generateFrontmatter } from "../team/document"; +import { isSessionWorthSharing } from "../team/formatter"; +import { callOllama } from "../team/ollama"; +import { ollamaChat, type OllamaTool } from "../ollama"; +import type { RawMessage } from "../team/formatter"; +import type { KnowledgeUnit } from "../team/types"; +import { + resolveEntity, + insertRelationship, + getRelationships, + findRelatedCandidates, + type RelationshipPredicate, +} from "./entities"; + +// ============================================================================= +// Types +// ============================================================================= + +export type ConsolidateOptions = { + minDensity?: number; + minRetrievals?: number; + minRelevance?: number; + minEntityReach?: number; + model?: string; + outputDir?: string; + author?: string; + sessionLimit?: number; + onProgress?: (msg: string) => void; + /** Also run the prune phase (dry-run by default — see pruneApply). */ + prune?: boolean; + /** Age threshold (days) for stale, never-promoted segmented units. Default 30. */ + pruneStaleDays?: number; + /** Actually delete/archive prune candidates. Without this, prune only reports candidates (dry-run). */ + pruneApply?: boolean; +}; + +export type PruneCandidate = { + id: string; + topic: string; + tier: "segmented" | "canonical"; + action: "delete" | "archive"; + reason: string; +}; + +export type ConsolidateResult = { + sessionsSegmented: number; + unitsStored: number; + unitsSkipped: number; + unitsPromoted: number; + /** Only set when options.prune is true. */ + unitsPruned?: number; + unitsArchived?: number; + pruneCandidates?: PruneCandidate[]; + errors: string[]; +}; + +// ============================================================================= +// Consolidation +// ============================================================================= + +export async function consolidateKnowledge( + db: Database, + options: ConsolidateOptions = {} +): Promise { + const author = options.author || AUTHOR; + const outputDir = options.outputDir || join(process.cwd(), SMRITI_DIR); + + const result: ConsolidateResult = { + sessionsSegmented: 0, + unitsStored: 0, + unitsSkipped: 0, + unitsPromoted: 0, + errors: [], + }; + + // =========================================================================== + // Segment phase: cheap Stage-1 extraction over dense, unsegmented sessions + // =========================================================================== + + const sessions = findUnsegmentedDenseSessions( + db, + options.minDensity ?? 0.5, + options.sessionLimit ?? 20 + ); + + for (const s of sessions) { + try { + const messages = getSessionMessages(db, s.session_id); + if (messages.length === 0) continue; + + const rawMessages: RawMessage[] = messages.map((m) => ({ + role: m.role, + content: m.content, + })); + + if (!isSessionWorthSharing(rawMessages)) continue; + + const segmentationResult = await segmentSession(db, s.session_id, rawMessages, { + model: options.model, + }); + result.sessionsSegmented++; + + for (const unit of segmentationResult.units) { + const contentHash = await hashContent( + JSON.stringify({ topic: unit.topic, category: unit.category, plainText: unit.plainText }) + ); + const inserted = insertKnowledgeUnit(db, unit, s.session_id, s.project_id, contentHash); + inserted ? result.unitsStored++ : result.unitsSkipped++; + + // Turn Stage 1's free-text entities into canonical "mentions" edges — + // pure post-processing of data already extracted, no extra LLM calls. + if (inserted) { + for (const rawEntity of unit.entities) { + const entityId = resolveEntity(db, rawEntity); + if (entityId) { + insertRelationship(db, "knowledge_unit", unit.id, "mentions", "entity", entityId, { + source: "extraction", + }); + } + } + } + } + } catch (err: any) { + result.errors.push(`segment ${s.session_id}: ${err.message}`); + } + } + + options.onProgress?.( + `segment phase: ${result.sessionsSegmented} sessions, ${result.unitsStored} units stored, ${result.unitsSkipped} skipped` + ); + + // =========================================================================== + // Promote phase: expensive Stage-2 polish for units that proved reuse + // =========================================================================== + + const knowledgeDir = join(outputDir, "knowledge"); + mkdirSync(knowledgeDir, { recursive: true }); + + const promotable = findPromotableUnits( + db, + options.minRetrievals ?? 3, + options.minRelevance ?? 8, + options.minEntityReach + ); + + for (const stored of promotable) { + try { + const unit: KnowledgeUnit = { + id: stored.id, + topic: stored.topic, + category: stored.category, + relevance: stored.relevance, + entities: stored.entities, + files: stored.files, + plainText: stored.plain_text, + lineRanges: stored.line_ranges, + }; + + // Bounded relationship inference: only runs if this unit shares a + // canonical entity with at least one other unit, and costs exactly one + // extra LLM call (same cost discipline as Stage 2) — persists what + // ollamaCheckConflicts previously only computed ephemerally. + const candidates = findRelatedCandidates(db, stored.id, 5); + if (candidates.length > 0) { + await inferRelationships(db, unit, candidates, options.model); + } + + const doc = await generateDocument(unit, stored.topic, { + model: options.model, + projectSmritiDir: outputDir, + author, + }); + + const categoryDir = join(knowledgeDir, doc.category.replaceAll("/", "-")); + mkdirSync(categoryDir, { recursive: true }); + const filePath = join(categoryDir, doc.filename); + + // Carry canonical entity ids + unit-to-unit edges into shared + // frontmatter. Unlike entity ids, these edges need no team-level + // canonicalization step — unit.id is already a portable UUID once + // shared (see src/team/document.ts's frontmatter `id` field), so + // syncTeamKnowledge can re-create them on a teammate's machine as-is. + const entityIds = getRelationships(db, { + subjectType: "knowledge_unit", + subjectId: stored.id, + predicate: "mentions", + objectType: "entity", + }).map((r) => r.object_id); + const outgoingEdges = getRelationships(db, { + subjectType: "knowledge_unit", + subjectId: stored.id, + }).filter((r) => r.predicate !== "mentions"); + + const fm = generateFrontmatter( + stored.session_id, + doc.unitId, + { + ...doc.frontmatter, + pipeline: "consolidated", + entity_ids: entityIds, + ...groupEdgesByPredicate(outgoingEdges), + }, + author, + stored.project_id || undefined + ); + await Bun.write(filePath, fm + "\n\n" + doc.markdown); + + const shareId = crypto.randomUUID().slice(0, 8); + const shareHash = await hashContent( + JSON.stringify({ content: doc.markdown, category: doc.category, entities: doc.frontmatter.entities }) + ); + + db.prepare( + `INSERT INTO smriti_shares (id, session_id, category_id, project_id, author, content_hash, unit_id, relevance_score, entities) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + shareId, + stored.session_id, + doc.category, + stored.project_id, + author, + shareHash, + doc.unitId, + stored.relevance, + JSON.stringify(stored.entities) + ); + + const relPath = `knowledge/${doc.category.replaceAll("/", "-")}/${doc.filename}`; + promoteKnowledgeUnit(db, stored.id, relPath, shareId); + result.unitsPromoted++; + } catch (err: any) { + result.errors.push(`promote ${stored.id}: ${err.message}`); + } + } + + options.onProgress?.(`promote phase: ${result.unitsPromoted} units promoted`); + + // =========================================================================== + // Prune phase: expire stale never-promoted units, archive superseded ones. + // Pure DB logic (age, retrieval_count, supersedes-edges are all already in + // SQLite by now) — no LLM call, dry-run by default. + // =========================================================================== + + if (options.prune) { + const pruneResult = await pruneKnowledge(db, { + outputDir, + pruneStaleDays: options.pruneStaleDays, + minRelevance: options.minRelevance, + dryRun: !options.pruneApply, + }); + result.unitsPruned = pruneResult.unitsPruned; + result.unitsArchived = pruneResult.unitsArchived; + result.pruneCandidates = pruneResult.pruneCandidates; + + options.onProgress?.( + pruneResult.pruneCandidates + ? `prune phase (dry-run): ${pruneResult.pruneCandidates.length} candidates — rerun with --yes to apply` + : `prune phase: ${pruneResult.unitsPruned} units deleted, ${pruneResult.unitsArchived} archived` + ); + } + + return result; +} + +// ============================================================================= +// Prune (expire stale segmented units, archive superseded canonical units) +// ============================================================================= + +export type PruneOptions = { + outputDir?: string; + pruneStaleDays?: number; + minRelevance?: number; + /** Report candidates without mutating the DB. Defaults to true — pass false to apply. */ + dryRun?: boolean; +}; + +export type PruneResult = { + unitsPruned: number; + unitsArchived: number; + pruneCandidates?: PruneCandidate[]; +}; + +export async function pruneKnowledge( + db: Database, + options: PruneOptions = {} +): Promise { + const dryRun = options.dryRun ?? true; + const outputDir = options.outputDir || join(process.cwd(), SMRITI_DIR); + const staleDays = options.pruneStaleDays ?? 30; + const minRelevance = options.minRelevance ?? 8; + + const stale = findStaleSegmentedUnits(db, staleDays, minRelevance); + const superseded = findSupersededCanonicalUnits(db); + + if (dryRun) { + const pruneCandidates: PruneCandidate[] = [ + ...stale.map((u) => ({ + id: u.id, + topic: u.topic, + tier: "segmented" as const, + action: "delete" as const, + reason: `stale segmented, 0 retrievals, relevance ${u.relevance} < ${minRelevance}`, + })), + ...superseded.map((u) => ({ + id: u.id, + topic: u.topic, + tier: "canonical" as const, + action: "archive" as const, + reason: `superseded by "${u.supersededByTopic}"`, + })), + ]; + return { unitsPruned: 0, unitsArchived: 0, pruneCandidates }; + } + + for (const u of stale) { + deleteKnowledgeUnit(db, u.id); + } + for (const u of superseded) { + archiveKnowledgeUnit(db, u.id, "superseded"); + await appendArchivedBanner(outputDir, u.canonical_doc_path, u.supersededByTopic); + } + + return { unitsPruned: stale.length, unitsArchived: superseded.length }; +} + +/** + * Prepend a short deprecation banner to an archived unit's canonical doc. + * The file itself is never deleted or moved — its path stays stable for + * anything already referencing it (team sync, a committed link) — only its + * content gains a notice pointing at the unit that superseded it. + */ +async function appendArchivedBanner( + outputDir: string, + docPath: string | null, + supersededByTopic: string +): Promise { + if (!docPath) return; + const filePath = join(outputDir, docPath); + const file = Bun.file(filePath); + if (!(await file.exists())) return; // doc already moved/removed outside Smriti — archive the DB row regardless + + const content = await file.text(); + const banner = `> **Archived** — superseded by "${supersededByTopic}".\n`; + const frontmatterMatch = content.match(/^---\n[\s\S]*?\n---\n/); + const updated = frontmatterMatch + ? content.slice(0, frontmatterMatch[0].length) + "\n" + banner + content.slice(frontmatterMatch[0].length) + : banner + "\n" + content; + + await Bun.write(filePath, updated); +} + +// ============================================================================= +// Relationship Inference (promote-time, LLM-gated) +// ============================================================================= + +const MAX_EXCERPT_CHARS = 800; + +export type RelationCandidate = { id: string; topic: string; category: string; plain_text: string }; +export type RelationGuess = { index: number; predicate: RelationshipPredicate | "none" }; + +function truncate(text: string, max: number): string { + return text.length > max ? text.slice(0, max) + "…" : text; +} + +function buildCandidateBlock(candidates: RelationCandidate[]): string { + return candidates + .map((c, i) => `[${i}] Topic: ${c.topic}\nCategory: ${c.category}\nContent: ${truncate(c.plain_text, MAX_EXCERPT_CHARS)}`) + .join("\n\n"); +} + +function buildComparisonPreamble( + unit: { topic: string; category: string; plainText: string }, + candidates: RelationCandidate[] +): string { + return `NEW UNIT +Topic: ${unit.topic} +Category: ${unit.category} +Content: ${truncate(unit.plainText, MAX_EXCERPT_CHARS)} + +CANDIDATES +${buildCandidateBlock(candidates)}`; +} + +// Brackets optional: models reliably get the index and predicate right but +// don't reliably reproduce "[i]" literally (observed: "RELATION 0: supersedes" +// instead of "RELATION [0]: supersedes") — a strict bracket requirement here +// silently drops otherwise-correct answers. +const RELATION_LINE = /RELATION\s*\[?(\d+)\]?:\s*(relatesTo|supersedes|contradicts|none)/gi; + +// Case-insensitive regex match -> canonical camelCase predicate (avoid a blind +// .toLowerCase() on the match, which would turn "relatesTo" into "relatesto"). +const PREDICATE_BY_LOWERCASE: Record = { + relatesto: "relatesTo", + supersedes: "supersedes", + contradicts: "contradicts", +}; + +/** + * Original approach: ask for free-text "RELATION [i]: predicate" lines and + * parse them with a regex. Kept only as the "before" baseline for the eval + * comparison against classifyRelationshipsToolCall — no longer wired into + * inferRelationships(). + */ +export async function classifyRelationshipsTextFormat( + unit: { topic: string; category: string; plainText: string }, + candidates: RelationCandidate[], + model?: string +): Promise { + const prompt = `You are comparing a NEW knowledge unit against CANDIDATE units that already mention at least one of the same topics/entities. + +${buildComparisonPreamble(unit, candidates)} + +For each candidate, decide the relationship of the NEW unit to it: +- relatesTo: related but neither replaces nor conflicts with the other +- supersedes: the NEW unit replaces/updates the candidate's guidance +- contradicts: the NEW unit conflicts with the candidate +- none: no meaningful relationship + +Respond with exactly one line per candidate, in this format: +RELATION [i]: relatesTo|supersedes|contradicts|none`; + + const response = await callOllama(prompt, { model }); + + const guesses: RelationGuess[] = []; + for (const match of response.matchAll(RELATION_LINE)) { + const index = Number(match[1]); + const raw = match[2]!.toLowerCase(); + const predicate = raw === "none" ? "none" : PREDICATE_BY_LOWERCASE[raw]; + if (!predicate || !candidates[index]) continue; + guesses.push({ index, predicate }); + } + + // A non-empty response that yields zero parsed lines almost always means + // the model drifted from the expected format, not that every candidate + // was genuinely unrelated — surface it instead of promoting in silence. + if (guesses.length === 0 && response.trim().length > 0) { + console.warn( + `classifyRelationshipsTextFormat: parsed 0 relation lines from a non-empty response — response may not match the expected format:`, + truncate(response, 300) + ); + } + + return guesses; +} + +const VALID_PREDICATES = new Set(["relatesTo", "supersedes", "contradicts", "none"]); + +const RECORD_RELATIONSHIPS_TOOL: OllamaTool = { + type: "function", + function: { + name: "record_relationships", + description: + "Record the relationship of the NEW knowledge unit to each CANDIDATE unit, one entry per candidate index.", + parameters: { + type: "object", + properties: { + relationships: { + type: "array", + items: { + type: "object", + properties: { + index: { + type: "integer", + description: "The candidate's [i] index as shown in the CANDIDATES list", + }, + predicate: { + type: "string", + enum: ["relatesTo", "supersedes", "contradicts", "none"], + description: + "relatesTo: related but neither replaces nor conflicts; supersedes: NEW unit replaces/updates the candidate; contradicts: NEW unit conflicts with the candidate; none: no meaningful relationship", + }, + }, + required: ["index", "predicate"], + }, + }, + }, + required: ["relationships"], + }, + }, +}; + +/** + * Ask the LLM to classify the NEW unit's relationship to each candidate via + * a native tool call instead of free-text lines — the model returns + * structured JSON directly, so there's no format to drift from and nothing + * to regex-parse. + */ +export async function classifyRelationshipsToolCall( + unit: { topic: string; category: string; plainText: string }, + candidates: RelationCandidate[], + model?: string +): Promise { + const prompt = `Compare the NEW knowledge unit against each CANDIDATE unit below, then call record_relationships with your assessment for every candidate index. + +${buildComparisonPreamble(unit, candidates)}`; + + const resp = await ollamaChat([{ role: "user", content: prompt }], { + model, + tools: [RECORD_RELATIONSHIPS_TOOL], + temperature: 0.1, + }); + + const call = resp.message.tool_calls?.find((c) => c.function.name === "record_relationships"); + if (!call) { + if (resp.message.content?.trim()) { + console.warn( + `classifyRelationshipsToolCall: model answered without calling record_relationships:`, + truncate(resp.message.content, 300) + ); + } + return []; + } + + const raw = call.function.arguments?.relationships; + if (!Array.isArray(raw)) return []; + + const guesses: RelationGuess[] = []; + for (const entry of raw) { + const index = Number((entry as any)?.index); + const predicate = (entry as any)?.predicate; + if (!Number.isInteger(index) || !candidates[index] || !VALID_PREDICATES.has(predicate)) continue; + guesses.push({ index, predicate }); + } + return guesses; +} + +/** + * Ask the LLM whether the unit being promoted relatesTo/supersedes/contradicts + * any of its entity-sharing candidates, and persist the answer as edges. + * Best-effort: a failure here (LLM down, unparseable response) is swallowed — + * it's enrichment on top of promotion, not a precondition for it. + */ +async function inferRelationships( + db: Database, + unit: KnowledgeUnit, + candidates: RelationCandidate[], + model?: string +): Promise { + try { + const guesses = await classifyRelationshipsToolCall(unit, candidates, model); + + for (const { index, predicate } of guesses) { + if (predicate === "none") continue; + const candidate = candidates[index]!; + + // Directional predicates shouldn't hold in both directions for the same + // pair. When two entity-sharing units are promoted in the same run, + // each independently asks "do I relate to/supersede the other" — if the + // candidate already asserted the reverse relation (e.g. its own + // promotion ran first in this batch), keep that one and skip the + // contradictory reverse edge rather than storing both. + const reverseAlreadyAsserted = getRelationships(db, { + subjectType: "knowledge_unit", + subjectId: candidate.id, + predicate, + objectType: "knowledge_unit", + objectId: unit.id, + }).length > 0; + if (reverseAlreadyAsserted) continue; + + insertRelationship(db, "knowledge_unit", unit.id, predicate, "knowledge_unit", candidate.id, { + source: "llm", + }); + } + } catch { + // Enrichment only — never block promotion on a failed/unparseable relation call. + } +} + +/** Group outgoing relationship edges by predicate into frontmatter-ready arrays of object unit ids. */ +function groupEdgesByPredicate( + edges: Array<{ predicate: string; object_id: string }> +): Record { + const grouped: Record = {}; + for (const edge of edges) { + (grouped[edge.predicate] ??= []).push(edge.object_id); + } + return grouped; +} diff --git a/src/learn/entities.ts b/src/learn/entities.ts new file mode 100644 index 0000000..470091d --- /dev/null +++ b/src/learn/entities.ts @@ -0,0 +1,232 @@ +/** + * learn/entities.ts - Canonical entity resolution + relationship triples + * + * RDF-inspired, not literal RDF: no URIs/Turtle/SPARQL. Subject/object are + * (type, id) pairs instead of global URIs, since Smriti is a local SQLite + * tool, not a web-facing linked-data endpoint. The useful ideas kept are: + * stable resource identity (so recurrence is detected across wording + * variance) and typed subject-predicate-object facts that survive team + * sharing (see src/team/config.ts's exportEntities/mergeEntities and + * src/team/document.ts's frontmatter for how these propagate org-wide). + * + * v1 entity resolution is exact-normalize only (case/whitespace/punctuation + * via slugify) — "JWT" and "jwt" merge, "JWT" and "JSON Web Token" do not. + * True synonym resolution needs semantic matching and is out of scope here. + */ + +import type { Database } from "bun:sqlite"; +import { slugify } from "../team/utils"; + +// ============================================================================= +// Types +// ============================================================================= + +export type EntityType = "technology" | "concept" | "file" | "pattern"; +export type RelationshipPredicate = "mentions" | "relatesTo" | "supersedes" | "contradicts"; +export type RelationshipSubjectType = "knowledge_unit" | "entity" | "session"; +export type RelationshipSource = "extraction" | "derived" | "llm"; + +export type StoredEntity = { + id: string; + label: string; + entity_type: EntityType; + aliases: string[]; + mention_count: number; + first_seen_at: string; +}; + +export type StoredRelationship = { + id: number; + subject_type: RelationshipSubjectType; + subject_id: string; + predicate: RelationshipPredicate; + object_type: RelationshipSubjectType; + object_id: string; + confidence: number; + source: RelationshipSource; + created_at: string; +}; + +type EntityRow = { + id: string; + label: string; + entity_type: string; + aliases: string; + mention_count: number; + first_seen_at: string; +}; + +function deserializeEntity(row: EntityRow): StoredEntity { + return { + ...row, + entity_type: row.entity_type as EntityType, + aliases: JSON.parse(row.aliases), + }; +} + +// ============================================================================= +// Entity Resolution +// ============================================================================= + +/** + * Resolve a raw, free-text entity label to a canonical entity id, creating + * the entity if it doesn't exist yet. Matching is exact-normalize (via + * slugify) — same case/whitespace variant collapses to one node; different + * wordings for the same concept do not (see module docstring). + */ +export function resolveEntity( + db: Database, + rawLabel: string, + entityType: EntityType = "concept" +): string | null { + const trimmed = rawLabel.trim(); + if (!trimmed) return null; + + const id = slugify(trimmed); + if (!id) return null; + + const existing = db + .prepare(`SELECT aliases FROM smriti_entities WHERE id = ?`) + .get(id) as { aliases: string } | null; + + if (existing) { + const aliases: string[] = JSON.parse(existing.aliases); + if (!aliases.includes(trimmed)) { + aliases.push(trimmed); + db.prepare( + `UPDATE smriti_entities SET aliases = ?, mention_count = mention_count + 1 WHERE id = ?` + ).run(JSON.stringify(aliases), id); + } else { + db.prepare(`UPDATE smriti_entities SET mention_count = mention_count + 1 WHERE id = ?`).run(id); + } + return id; + } + + db.prepare( + `INSERT INTO smriti_entities (id, label, entity_type, aliases, mention_count) + VALUES (?, ?, ?, ?, 1)` + ).run(id, trimmed, entityType, JSON.stringify([trimmed])); + return id; +} + +export function getEntity(db: Database, id: string): StoredEntity | null { + const row = db.prepare(`SELECT * FROM smriti_entities WHERE id = ?`).get(id) as EntityRow | null; + return row ? deserializeEntity(row) : null; +} + +/** Look up an entity by exact id, or by slugified/label match against a raw query string. */ +export function findEntity(db: Database, query: string): StoredEntity | null { + const bySlug = getEntity(db, slugify(query)); + if (bySlug) return bySlug; + + const row = db + .prepare(`SELECT * FROM smriti_entities WHERE LOWER(label) = LOWER(?)`) + .get(query.trim()) as EntityRow | null; + return row ? deserializeEntity(row) : null; +} + +/** Knowledge units that `mentions` a given canonical entity — the display side of `smriti graph `. */ +export function getUnitsForEntity( + db: Database, + entityId: string +): Array<{ id: string; topic: string; category: string; relevance: number; tier: string; retrieval_count: number }> { + return db + .prepare( + `SELECT ku.id, ku.topic, ku.category, ku.relevance, ku.tier, ku.retrieval_count + FROM smriti_relationships r + JOIN smriti_knowledge_units ku ON ku.id = r.subject_id + WHERE r.subject_type = 'knowledge_unit' AND r.object_type = 'entity' + AND r.predicate = 'mentions' AND r.object_id = ? AND ku.tier != 'archived' + ORDER BY ku.retrieval_count DESC, ku.relevance DESC` + ) + .all(entityId) as Array<{ + id: string; topic: string; category: string; relevance: number; tier: string; retrieval_count: number; + }>; +} + +export function listEntities(db: Database, limit?: number): StoredEntity[] { + const rows = ( + limit + ? db.prepare(`SELECT * FROM smriti_entities ORDER BY mention_count DESC LIMIT ?`).all(limit) + : db.prepare(`SELECT * FROM smriti_entities ORDER BY mention_count DESC`).all() + ) as EntityRow[]; + return rows.map(deserializeEntity); +} + +// ============================================================================= +// Relationship Triples +// ============================================================================= + +/** Insert a (subject, predicate, object) triple. Deduped via the table's UNIQUE constraint. */ +export function insertRelationship( + db: Database, + subjectType: RelationshipSubjectType, + subjectId: string, + predicate: RelationshipPredicate, + objectType: RelationshipSubjectType, + objectId: string, + options: { confidence?: number; source?: RelationshipSource } = {} +): void { + db.prepare( + `INSERT OR IGNORE INTO smriti_relationships + (subject_type, subject_id, predicate, object_type, object_id, confidence, source) + VALUES (?, ?, ?, ?, ?, ?, ?)` + ).run( + subjectType, + subjectId, + predicate, + objectType, + objectId, + options.confidence ?? 1.0, + options.source ?? "extraction" + ); +} + +export type TriplePattern = { + subjectType?: RelationshipSubjectType; + subjectId?: string; + predicate?: RelationshipPredicate; + objectType?: RelationshipSubjectType; + objectId?: string; +}; + +/** Single-pattern triple lookup — the basic-graph-pattern piece of SPARQL, simplified to one triple at a time. */ +export function getRelationships(db: Database, pattern: TriplePattern): StoredRelationship[] { + const conditions: string[] = []; + const params: any[] = []; + + if (pattern.subjectType) { conditions.push("subject_type = ?"); params.push(pattern.subjectType); } + if (pattern.subjectId) { conditions.push("subject_id = ?"); params.push(pattern.subjectId); } + if (pattern.predicate) { conditions.push("predicate = ?"); params.push(pattern.predicate); } + if (pattern.objectType) { conditions.push("object_type = ?"); params.push(pattern.objectType); } + if (pattern.objectId) { conditions.push("object_id = ?"); params.push(pattern.objectId); } + + const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : ""; + return db.prepare(`SELECT * FROM smriti_relationships ${where}`).all(...params) as StoredRelationship[]; +} + +/** + * Find other knowledge units that `mentions` at least one of the same + * canonical entities as `unitId` — the candidate set for promote-time + * LLM relationship inference (bounded, so that call stays cheap). + */ +export function findRelatedCandidates( + db: Database, + unitId: string, + limit: number = 5 +): Array<{ id: string; topic: string; category: string; plain_text: string }> { + return db + .prepare( + `SELECT DISTINCT ku.id, ku.topic, ku.category, ku.plain_text + FROM smriti_relationships r1 + JOIN smriti_relationships r2 + ON r1.object_id = r2.object_id + AND r2.object_type = 'entity' AND r2.predicate = 'mentions' + JOIN smriti_knowledge_units ku ON ku.id = r2.subject_id + WHERE r1.subject_type = 'knowledge_unit' AND r1.subject_id = ? + AND r1.object_type = 'entity' AND r1.predicate = 'mentions' + AND r2.subject_id != r1.subject_id + LIMIT ?` + ) + .all(unitId, limit) as Array<{ id: string; topic: string; category: string; plain_text: string }>; +} diff --git a/src/memory.ts b/src/memory.ts index ff86e8f..f5045c1 100644 --- a/src/memory.ts +++ b/src/memory.ts @@ -257,6 +257,44 @@ export function clearAllSessions(db: Database, hard: boolean = false): number { } } +/** + * Remove content_vectors/vectors_vec rows whose hash is no longer referenced + * by any memory message or active QMD document. Scoped deletion — unlike + * QMD's own cleanupOrphanedVectors (which only checks `documents` and would + * wipe every memory-message embedding, since messages aren't rows in + * `documents`). Called after a hard session delete; a no-op (returns 0) when + * the vector/document tables aren't present (e.g. sqlite-vec unavailable, or + * a minimal test schema that skipped createStore()). + */ +export function cleanupOrphanedMemoryVectors(db: Database): number { + try { + db.prepare(`SELECT 1 FROM vectors_vec LIMIT 0`).get(); + db.prepare(`SELECT 1 FROM documents LIMIT 0`).get(); + db.prepare(`SELECT 1 FROM content_vectors LIMIT 0`).get(); + } catch { + return 0; + } + + const orphanWhere = ` + NOT EXISTS (SELECT 1 FROM memory_messages m WHERE m.hash = content_vectors.hash) + AND NOT EXISTS (SELECT 1 FROM documents d WHERE d.hash = content_vectors.hash AND d.active = 1) + `; + + const { c } = db + .prepare(`SELECT COUNT(*) as c FROM content_vectors WHERE ${orphanWhere}`) + .get() as { c: number }; + if (c === 0) return 0; + + db.exec(` + DELETE FROM vectors_vec WHERE hash_seq IN ( + SELECT content_vectors.hash || '_' || content_vectors.seq FROM content_vectors WHERE ${orphanWhere} + ) + `); + db.exec(`DELETE FROM content_vectors WHERE ${orphanWhere}`); + + return c; +} + // ============================================================================= // Message CRUD // ============================================================================= @@ -828,23 +866,31 @@ export async function recallMemories( } } - // Blend density scores into recall scores — dense sessions rank higher + // Blend density scores into recall scores — dense sessions rank higher. + // smriti_session_meta is a Smriti-layer table, not a QMD core one — this + // file is meant to stay usable against a bare QMD store (e.g. + // scripts/bench-qmd.ts), so a missing table degrades gracefully instead + // of throwing, same as the vector-search fallback above. if (dedupedResults.length > 0) { - const sessionIds = dedupedResults.map((r) => r.session_id); - const placeholders = sessionIds.map(() => "?").join(","); - const densityRows = (db as any) - .prepare( - `SELECT session_id, COALESCE(density_score, 0) as density_score - FROM smriti_session_meta WHERE session_id IN (${placeholders})` - ) - .all(...sessionIds) as { session_id: string; density_score: number }[]; - const densityMap = new Map(densityRows.map((r) => [r.session_id, r.density_score])); + try { + const sessionIds = dedupedResults.map((r) => r.session_id); + const placeholders = sessionIds.map(() => "?").join(","); + const densityRows = (db as any) + .prepare( + `SELECT session_id, COALESCE(density_score, 0) as density_score + FROM smriti_session_meta WHERE session_id IN (${placeholders})` + ) + .all(...sessionIds) as { session_id: string; density_score: number }[]; + const densityMap = new Map(densityRows.map((r) => [r.session_id, r.density_score])); - for (const r of dedupedResults) { - const ds = densityMap.get(r.session_id) ?? 0; - r.score = r.score * 0.8 + ds * 0.2; + for (const r of dedupedResults) { + const ds = densityMap.get(r.session_id) ?? 0; + r.score = r.score * 0.8 + ds * 0.2; + } + dedupedResults.sort((a, b) => b.score - a.score); + } catch { + // smriti_session_meta doesn't exist (bare QMD store) — skip blending. } - dedupedResults.sort((a, b) => b.score - a.score); } const results = dedupedResults.slice(0, limit); diff --git a/src/ollama.ts b/src/ollama.ts index 833315c..f316209 100644 --- a/src/ollama.ts +++ b/src/ollama.ts @@ -6,29 +6,40 @@ * * Config via env: * OLLAMA_HOST - Ollama server URL (default: http://127.0.0.1:11434) - * QMD_MEMORY_MODEL - Model for summarization/synthesis (default: qwen3:8b-tuned) + * QMD_MEMORY_MODEL - Model for summarization/synthesis (required, no default) */ -// ============================================================================= -// Configuration -// ============================================================================= - -const OLLAMA_HOST = Bun.env.OLLAMA_HOST || "http://127.0.0.1:11434"; -const DEFAULT_MEMORY_MODEL = Bun.env.QMD_MEMORY_MODEL || "qwen3:8b-tuned"; +import { OLLAMA_HOST, requireOllamaModel } from "./config"; // ============================================================================= // Types // ============================================================================= +export type OllamaToolCall = { + id?: string; + function: { name: string; arguments: Record }; +}; + export type OllamaChatMessage = { - role: "system" | "user" | "assistant"; + role: "system" | "user" | "assistant" | "tool"; content: string; + tool_calls?: OllamaToolCall[]; +}; + +export type OllamaTool = { + type: "function"; + function: { + name: string; + description: string; + parameters: Record; + }; }; export type OllamaChatOptions = { model?: string; temperature?: number; maxTokens?: number; + tools?: OllamaTool[]; }; export type OllamaChatResponse = { @@ -51,7 +62,7 @@ export async function ollamaChat( messages: OllamaChatMessage[], options: OllamaChatOptions = {} ): Promise { - const model = options.model || DEFAULT_MEMORY_MODEL; + const model = requireOllamaModel(options.model); const resp = await fetch(`${OLLAMA_HOST}/api/chat`, { method: "POST", headers: { "Content-Type": "application/json" }, @@ -59,6 +70,7 @@ export async function ollamaChat( model, messages, stream: false, + ...(options.tools && { tools: options.tools }), options: { ...(options.temperature !== undefined && { temperature: options.temperature }), ...(options.maxTokens !== undefined && { num_predict: options.maxTokens }), @@ -288,5 +300,3 @@ export async function ollamaHealthCheck(): Promise<{ }; } } - -export { DEFAULT_MEMORY_MODEL, OLLAMA_HOST }; diff --git a/src/qmd.ts b/src/qmd.ts index d474c90..6f4f6c8 100644 --- a/src/qmd.ts +++ b/src/qmd.ts @@ -17,6 +17,9 @@ export { importTranscript, initializeMemoryTables, createSession, + deleteSession, + clearAllSessions, + cleanupOrphanedMemoryVectors, } from "./memory"; export { hashContent } from "../qmd/src/store"; diff --git a/src/search/recall.ts b/src/search/recall.ts index 1e219a4..f94fe7b 100644 --- a/src/search/recall.ts +++ b/src/search/recall.ts @@ -9,6 +9,7 @@ import { DEFAULT_RECALL_LIMIT, OLLAMA_HOST, OLLAMA_MODEL } from "../config"; import { recallMemories, ollamaRecall } from "../qmd"; import { searchFiltered, type SearchFilters, type SearchResult } from "./index"; import { getQmdStore } from "../store"; +import { incrementRetrievalCount } from "../db"; // ============================================================================= // Types @@ -27,6 +28,24 @@ export type RecallResult = { synthesis?: string; }; +// ============================================================================= +// Retrieval Tracking +// ============================================================================= + +/** + * Best-effort bump of retrieval_count for any consolidated knowledge units + * belonging to the recalled sessions. Never lets a tracking failure break recall. + */ +function trackRetrieval(db: Database, results: SearchResult[]): void { + try { + for (const sessionId of new Set(results.map((r) => r.session_id).filter(Boolean))) { + incrementRetrievalCount(db, sessionId); + } + } catch { + // Never let this break recall. + } +} + // ============================================================================= // Filtered Recall // ============================================================================= @@ -58,6 +77,7 @@ export async function recall( if (options.synthesize && storeResults.length > 0) { synthesis = await synthesizeResults(query, storeResults, options); } + trackRetrieval(db, storeResults); return { results: storeResults, synthesis }; } @@ -70,6 +90,7 @@ export async function recall( fast: options.fast, intent: rerankIntent, }); + trackRetrieval(db, qmdResult.results); return { results: qmdResult.results, synthesis: qmdResult.synthesis, @@ -102,6 +123,7 @@ export async function recall( synthesis = await synthesizeResults(query, deduped, options); } + trackRetrieval(db, deduped); return { results: deduped, synthesis }; } diff --git a/src/team/config.ts b/src/team/config.ts index 8dddcbc..13c0efe 100644 --- a/src/team/config.ts +++ b/src/team/config.ts @@ -17,9 +17,17 @@ export type CustomCategoryDef = { description?: string; }; +export type CustomEntityDef = { + id: string; + label: string; + entity_type: string; + aliases: string[]; +}; + export type SmritiConfig = { version: number; categories?: CustomCategoryDef[]; + entities?: CustomEntityDef[]; allowedCategories?: string[]; autoSync?: boolean; }; @@ -115,3 +123,73 @@ export function exportCustomCategories(db: Database): CustomCategoryDef[] { ...(r.description ? { description: r.description } : {}), })); } + +// ============================================================================= +// Entity Merge — team/org propagation for smriti_entities +// +// Same reasoning as categories: an entity's canonical id is only meaningful +// if every teammate's machine agrees on it. .smriti/config.json is the +// git-committed source of truth each local smriti_entities table converges +// toward on every share/sync — the same role a published vocabulary plays +// for literal RDF, minus the URIs. +// ============================================================================= + +/** + * Upsert entities from config into the local DB. Matches first by id, then + * falls back to a normalized-label match (so two machines that independently + * minted different ids for the same concept still converge once either + * syncs the shared file). Unions aliases on conflict rather than overwriting. + * Returns count of newly created entities. + */ +export function mergeEntities(db: Database, entities: CustomEntityDef[]): number { + if (entities.length === 0) return 0; + + let created = 0; + for (const entity of entities) { + const existingById = db + .prepare(`SELECT id, aliases FROM smriti_entities WHERE id = ?`) + .get(entity.id) as { id: string; aliases: string } | null; + + if (existingById) { + const aliases = new Set(JSON.parse(existingById.aliases)); + for (const a of entity.aliases) aliases.add(a); + db.prepare(`UPDATE smriti_entities SET aliases = ? WHERE id = ?`) + .run(JSON.stringify([...aliases]), entity.id); + continue; + } + + const normalizedLabel = entity.label.trim().toLowerCase(); + const existingByLabel = db + .prepare(`SELECT id, aliases FROM smriti_entities WHERE LOWER(label) = ?`) + .get(normalizedLabel) as { id: string; aliases: string } | null; + + if (existingByLabel) { + const aliases = new Set(JSON.parse(existingByLabel.aliases)); + for (const a of entity.aliases) aliases.add(a); + db.prepare(`UPDATE smriti_entities SET aliases = ? WHERE id = ?`) + .run(JSON.stringify([...aliases]), existingByLabel.id); + continue; + } + + db.prepare( + `INSERT INTO smriti_entities (id, label, entity_type, aliases, mention_count) + VALUES (?, ?, ?, ?, 0)` + ).run(entity.id, entity.label, entity.entity_type, JSON.stringify(entity.aliases)); + created++; + } + return created; +} + +/** Query smriti_entities and return as config defs for export to .smriti/config.json. */ +export function exportEntities(db: Database): CustomEntityDef[] { + const rows = db + .prepare(`SELECT id, label, entity_type, aliases FROM smriti_entities`) + .all() as Array<{ id: string; label: string; entity_type: string; aliases: string }>; + + return rows.map((r) => ({ + id: r.id, + label: r.label, + entity_type: r.entity_type, + aliases: JSON.parse(r.aliases), + })); +} diff --git a/src/team/ollama.ts b/src/team/ollama.ts index 6ea04f7..655347c 100644 --- a/src/team/ollama.ts +++ b/src/team/ollama.ts @@ -5,7 +5,7 @@ * Used by segment.ts (Stage 1) and document.ts (Stage 2). */ -import { OLLAMA_HOST, OLLAMA_MODEL } from "../config"; +import { OLLAMA_HOST, requireOllamaModel } from "../config"; export type OllamaOptions = { model?: string; @@ -28,7 +28,7 @@ export async function callOllama( prompt: string, options: OllamaOptions = {} ): Promise { - const model = options.model || OLLAMA_MODEL; + const model = requireOllamaModel(options.model); const temperature = options.temperature ?? 0.7; const timeout = options.timeout ?? DEFAULT_TIMEOUT; const maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES; diff --git a/src/team/reflect.ts b/src/team/reflect.ts index 32ca106..6612709 100644 --- a/src/team/reflect.ts +++ b/src/team/reflect.ts @@ -10,7 +10,7 @@ * 2. src/team/prompts/share-reflect.md (built-in default) */ -import { OLLAMA_HOST, OLLAMA_MODEL } from "../config"; +import { OLLAMA_HOST, requireOllamaModel } from "../config"; import { join, dirname } from "path"; import { fileURLToPath } from "url"; import type { RawMessage } from "./formatter"; @@ -222,7 +222,7 @@ export async function synthesizeSession( const template = await loadPromptTemplate(options.projectSmritiDir); const prompt = template.replace("{{conversation}}", conversation); - const model = options.model || OLLAMA_MODEL; + const model = requireOllamaModel(options.model); const timeout = options.timeout || 120_000; const controller = new AbortController(); diff --git a/src/team/share.ts b/src/team/share.ts index 06c44e5..22adef8 100644 --- a/src/team/share.ts +++ b/src/team/share.ts @@ -10,7 +10,7 @@ import { SMRITI_DIR, AUTHOR } from "../config"; import { hashContent } from "../qmd"; import { existsSync, mkdirSync } from "fs"; import { join } from "path"; -import { readConfig, writeConfig, exportCustomCategories } from "./config"; +import { readConfig, writeConfig, exportCustomCategories, exportEntities } from "./config"; import { formatSessionAsFallback, isSessionWorthSharing, @@ -140,7 +140,7 @@ function querySessions( } /** Get messages for a session */ -function getSessionMessages( +export function getSessionMessages( db: Database, sessionId: string ): Array<{ @@ -178,15 +178,17 @@ async function writeManifest( const fullManifest = [...existingManifest, ...newEntries]; await Bun.write(indexPath, JSON.stringify(fullManifest, null, 2)); - // Write config — always update with latest custom categories + // Write config — always update with latest custom categories + canonical entities const existing = readConfig(outputDir); const customCategories = db ? exportCustomCategories(db) : []; + const entities = db ? exportEntities(db) : []; const config = { ...existing, - version: customCategories.length > 0 ? 2 : (existing.version ?? 1), + version: customCategories.length > 0 || entities.length > 0 ? 2 : (existing.version ?? 1), allowedCategories: existing.allowedCategories ?? ["*"], autoSync: existing.autoSync ?? false, ...(customCategories.length > 0 ? { categories: customCategories } : {}), + ...(entities.length > 0 ? { entities } : {}), }; await writeConfig(outputDir, config); diff --git a/src/team/sync.ts b/src/team/sync.ts index fdb9d5a..b4e51cc 100644 --- a/src/team/sync.ts +++ b/src/team/sync.ts @@ -9,7 +9,8 @@ import type { Database } from "bun:sqlite"; import { SMRITI_DIR } from "../config"; import { addMessage, hashContent } from "../qmd"; import { join } from "path"; -import { readConfig, mergeCategories } from "./config"; +import { readConfig, mergeCategories, mergeEntities } from "./config"; +import { insertRelationship, type RelationshipPredicate } from "../learn/entities"; // ============================================================================= // Types @@ -26,6 +27,7 @@ export type SyncResult = { skipped: number; errors: string[]; categoriesImported: number; + entitiesImported: number; }; // ============================================================================= @@ -116,6 +118,7 @@ export async function syncTeamKnowledge( skipped: 0, errors: [], categoriesImported: 0, + entitiesImported: 0, }; // Determine input directory @@ -144,11 +147,16 @@ export async function syncTeamKnowledge( ).map((r) => r.content_hash) ); - // Import custom categories from config.json (v2+) before scanning files + // Import custom categories + canonical entities from config.json (v2+) + // before scanning files — entities must exist locally before per-file + // "mentions"/relationship edges below can reference them. const config = readConfig(inputDir); if (config.categories && config.categories.length > 0) { result.categoriesImported = mergeCategories(db, config.categories); } + if (config.entities && config.entities.length > 0) { + result.entitiesImported = mergeEntities(db, config.entities); + } // Scan for markdown files const knowledgeDir = join(inputDir, "knowledge"); @@ -175,10 +183,11 @@ export async function syncTeamKnowledge( continue; } - // Segmented pipeline docs don't have **user**/**assistant** patterns; - // treat the whole body as a single assistant message. - const isSegmented = meta.pipeline === "segmented"; - const messages = isSegmented + // Segmented and consolidated pipeline docs don't have + // **user**/**assistant** patterns; treat the whole body as a single + // assistant message. + const isSingleMessageDoc = meta.pipeline === "segmented" || meta.pipeline === "consolidated"; + const messages = isSingleMessageDoc ? [{ role: "assistant", content: body.trim() }] : extractMessages(body); @@ -242,6 +251,28 @@ export async function syncTeamKnowledge( contentHash ); + // Re-create relationship edges from frontmatter. Unlike entities, + // these need no canonicalization: unit ids are portable UUIDs + // already shared as `sessionId` above, so edges reference the same + // node on every machine that imports this file. + const asArray = (v: string | string[] | undefined): string[] => + v === undefined ? [] : Array.isArray(v) ? v : [v]; + + for (const entityId of asArray(meta.entity_ids)) { + if (!entityId) continue; + insertRelationship(db, "knowledge_unit", sessionId, "mentions", "entity", entityId, { + source: "extraction", + }); + } + for (const predicate of ["relatesTo", "supersedes", "contradicts"] as RelationshipPredicate[]) { + for (const objectId of asArray(meta[predicate])) { + if (!objectId) continue; + insertRelationship(db, "knowledge_unit", sessionId, predicate, "knowledge_unit", objectId, { + source: "llm", + }); + } + } + result.imported++; } catch (err: any) { result.errors.push(`${match}: ${err.message}`); diff --git a/test/eval/fixtures/auth-migration.ts b/test/eval/fixtures/auth-migration.ts new file mode 100644 index 0000000..696c9e1 --- /dev/null +++ b/test/eval/fixtures/auth-migration.ts @@ -0,0 +1,58 @@ +import type { RecallScenario } from "./types"; + +/** Cross-session recall over time, plus precision against a same-project, same-vocabulary distractor. */ +export const AUTH_MIGRATION: RecallScenario = { + name: "auth: JWT -> session cookie migration, recalled later", + project: "webapp", + sessions: [ + { + id: "auth-s1", + daysAgo: 14, + messages: [ + { role: "user", content: "We're getting complaints that JWTs can't be revoked on logout. Should we move to session cookies?" }, + { role: "assistant", content: "Session cookies with server-side storage let you revoke instantly. JWTs need a blocklist, which defeats the point of being stateless." }, + { role: "user", content: "Let's switch. Use server-side session cookies, drop JWT for auth entirely." }, + { role: "assistant", content: "Decision: replaced JWT-based session tokens with server-side session cookies stored in Redis, keyed by session id, TTL 24h." }, + ], + }, + { + id: "auth-s2", + daysAgo: 10, + messages: [ + { role: "user", content: "Session cookie migration is live. Any edge cases with mobile clients that can't set cookies?" }, + { role: "assistant", content: "Mobile WebViews sometimes block third-party cookies; we added a fallback bearer-token header for the mobile app only." }, + ], + }, + { + id: "auth-s3-distractor", + daysAgo: 30, + messages: [ + { role: "user", content: "What's our password hashing algorithm for the auth system?" }, + { role: "assistant", content: "We use bcrypt with cost factor 12 for password hashing, unrelated to session management." }, + ], + }, + ], + probes: [ + { + // FTS5 MATCH is an AND of all terms within one message — terms must + // be literally present, not natural-language phrasing (see msg2: + // "Use server-side session cookies, drop JWT for auth entirely."). + query: "JWT session cookies auth", + description: "core decision recall — must surface the migration session", + expectHitSessionIds: ["auth-s1"], + expectHitSubstrings: ["server-side session cookies"], + expectMissSessionIds: ["auth-s3-distractor"], + }, + { + query: "mobile cookie migration", + description: "follow-up detail in a later session — recall must surface s2, not just s1", + expectHitSessionIds: ["auth-s2"], + }, + { + query: "password hashing algorithm", + description: "distractor probe — different topic sharing the 'auth' vocabulary; must not pull in s1/s2", + expectHitSessionIds: ["auth-s3-distractor"], + expectMissSessionIds: ["auth-s1", "auth-s2"], + }, + ], +}; diff --git a/test/eval/fixtures/density-recency.ts b/test/eval/fixtures/density-recency.ts new file mode 100644 index 0000000..1e3acc4 --- /dev/null +++ b/test/eval/fixtures/density-recency.ts @@ -0,0 +1,41 @@ +import type { RecallScenario } from "./types"; + +/** + * Two sessions with identical content (so their BM25/RRF scores tie exactly) + * but very different density_score — recallMemories blends density into the + * final score 80/20, so at topK=1 only the denser session should survive. + * Requires useRecallMemories: true, since the project-filtered searchFiltered + * path never touches density scoring. + */ +export const DENSITY_BLENDING: RecallScenario = { + name: "density blending breaks a BM25 tie", + project: "backend", + sessions: [ + { + id: "density-high", + densityScore: 0.9, + messages: [ + { role: "user", content: "We should switch to using a message queue for background job processing." }, + { role: "assistant", content: "Agreed — moving long-running work off the request path avoids timeouts." }, + ], + }, + { + id: "density-low", + densityScore: 0.05, + messages: [ + { role: "user", content: "We should switch to using a message queue for background job processing." }, + { role: "assistant", content: "Agreed — moving long-running work off the request path avoids timeouts." }, + ], + }, + ], + probes: [ + { + query: "message queue background job processing", + description: "BM25-tied content, density_score must break the tie toward the denser session", + expectHitSessionIds: ["density-high"], + expectMissSessionIds: ["density-low"], + topK: 1, + useRecallMemories: true, + }, + ], +}; diff --git a/test/eval/fixtures/deploy-pipeline.ts b/test/eval/fixtures/deploy-pipeline.ts new file mode 100644 index 0000000..2065e9d --- /dev/null +++ b/test/eval/fixtures/deploy-pipeline.ts @@ -0,0 +1,42 @@ +import type { RecallScenario } from "./types"; + +/** + * Single-session recall, scoped by project. A second session shares almost + * identical vocabulary ("deploys are flaky", "CI pipeline") but lives under + * a different project — the probe (scoped to "api-service") must not pull + * it in, proving project filtering isolates results rather than just + * favoring topical relevance. + */ +export const DEPLOY_PIPELINE: RecallScenario = { + name: "deploy: CI pipeline decision, isolated by project", + project: "api-service", + sessions: [ + { + id: "deploy-s1", + messages: [ + { role: "user", content: "Our deploys are flaky. What's causing the intermittent CI pipeline failures?" }, + { role: "assistant", content: "Flaky tests were racing against a shared test database. Switched the pipeline to spin up an isolated Postgres container per CI job." }, + { role: "user", content: "Good, let's also cache node_modules between runs to speed things up." }, + { role: "assistant", content: "Added actions/cache keyed on the lockfile hash — pipeline runtime dropped from 8 minutes to 3." }, + ], + }, + { + id: "deploy-s2-other-project", + project: "frontend-app", + messages: [ + { role: "user", content: "Our deploys are flaky too — the CI pipeline times out on the frontend build." }, + { role: "assistant", content: "The frontend bundle got too large for the default Vercel build timeout; raised it and split the vendor chunk." }, + ], + }, + ], + probes: [ + { + // FTS5 MATCH is an AND of all terms within one message — literal terms only. + query: "CI pipeline flaky", + description: "single-session recall scoped to api-service — must not pull in the same-vocabulary frontend-app session", + expectHitSessionIds: ["deploy-s1"], + expectHitSubstrings: ["isolated Postgres container", "shared test database"], + expectMissSessionIds: ["deploy-s2-other-project"], + }, + ], +}; diff --git a/test/eval/fixtures/index.ts b/test/eval/fixtures/index.ts new file mode 100644 index 0000000..0db32bc --- /dev/null +++ b/test/eval/fixtures/index.ts @@ -0,0 +1,16 @@ +import { AUTH_MIGRATION } from "./auth-migration"; +import { DEPLOY_PIPELINE } from "./deploy-pipeline"; +import { DENSITY_BLENDING } from "./density-recency"; +import { SEMANTIC_CACHING } from "./semantic-caching"; +import type { RecallScenario } from "./types"; + +/** BM25-only scenarios — deterministic, no embeddings needed. Safe for CI. */ +export const CI_SCENARIOS: RecallScenario[] = [AUTH_MIGRATION, DEPLOY_PIPELINE, DENSITY_BLENDING]; + +/** Needs a live embedding backend — manual-only (see test/eval/recall-quality.eval.ts). */ +export const QUALITY_ONLY_SCENARIOS: RecallScenario[] = [SEMANTIC_CACHING]; + +/** Full set — quality mode runs all of these; CI mode filters to CI_SCENARIOS. */ +export const ALL_SCENARIOS: RecallScenario[] = [...CI_SCENARIOS, ...QUALITY_ONLY_SCENARIOS]; + +export type { RecallScenario, FixtureSession, Probe } from "./types"; diff --git a/test/eval/fixtures/run.ts b/test/eval/fixtures/run.ts new file mode 100644 index 0000000..04f59f0 --- /dev/null +++ b/test/eval/fixtures/run.ts @@ -0,0 +1,27 @@ +/** + * test/eval/fixtures/run.ts - Shared probe runner for the recall-quality + * harness (Tier 1 CI test and Tier 2 manual eval both call this). + */ + +import type { Database } from "bun:sqlite"; +import { recall } from "../../../src/search/recall"; +import { scoreProbe, type ProbeScore } from "./score"; +import type { Probe, RecallScenario } from "./types"; + +export const DEFAULT_TOP_K = 5; + +export async function runProbe( + db: Database, + scenario: RecallScenario, + probe: Probe, + options: { fast: boolean } +): Promise<{ score: ProbeScore; latencyMs: number; sources: string[] }> { + const limit = probe.topK ?? DEFAULT_TOP_K; + const started = performance.now(); + const { results } = probe.useRecallMemories + ? await recall(db, probe.query, { fast: options.fast, limit }) + : await recall(db, probe.query, { project: probe.project ?? scenario.project, fast: options.fast, limit }); + const latencyMs = performance.now() - started; + const score = scoreProbe(results, probe); + return { score, latencyMs, sources: [...new Set(results.map((r) => r.source))] }; +} diff --git a/test/eval/fixtures/score.ts b/test/eval/fixtures/score.ts new file mode 100644 index 0000000..3ba2535 --- /dev/null +++ b/test/eval/fixtures/score.ts @@ -0,0 +1,61 @@ +/** + * test/eval/fixtures/score.ts - Scoring for recall-quality probes. + * + * Precision is computed only against a probe's explicit expectMissSessionIds + * (known distractors), not exhaustively against everything else — the same + * "narrow but honest" approach test/eval/relation-inference.eval.ts uses for + * exact-match grading. We can't exhaustively label every session a probe + * shouldn't match, so we only assert on the ones we deliberately planted. + */ + +import type { Probe } from "./types"; + +export type ProbeResult = { session_id: string; content: string }; + +export type ProbeScore = { + recall: number; // |expected ∩ hit| / |expected|, 1 if no expected hits defined + precision: number | null; // 1 - |miss ∩ hit| / |hit|, null if the probe defines no distractors + substringOk: boolean; // true if no expectHitSubstrings, or one matched a retrieved expected-hit row + pass: boolean; +}; + +export function scoreProbe(results: ProbeResult[], probe: Probe): ProbeScore { + const hitIds = new Set(results.map((r) => r.session_id)); + + const expected = probe.expectHitSessionIds; + const hitCount = expected.filter((id) => hitIds.has(id)).length; + const recall = expected.length ? hitCount / expected.length : 1; + + let precision: number | null = null; + if (probe.expectMissSessionIds && probe.expectMissSessionIds.length > 0) { + const missHits = probe.expectMissSessionIds.filter((id) => hitIds.has(id)).length; + precision = hitIds.size > 0 ? 1 - missHits / hitIds.size : 1; + } + + let substringOk = true; + if (probe.expectHitSubstrings && probe.expectHitSubstrings.length > 0) { + const expectedRows = results.filter((r) => expected.includes(r.session_id)); + substringOk = expectedRows.some((r) => + probe.expectHitSubstrings!.some((s) => r.content.toLowerCase().includes(s.toLowerCase())) + ); + } + + const pass = recall === 1 && (precision === null || precision === 1) && substringOk; + return { recall, precision, substringOk, pass }; +} + +export function summarizeScores(scores: ProbeScore[]): { + total: number; + passed: number; + avgRecall: number; + avgPrecision: number | null; +} { + const total = scores.length; + const passed = scores.filter((s) => s.pass).length; + const avgRecall = total ? scores.reduce((sum, s) => sum + s.recall, 0) / total : 1; + const withPrecision = scores.filter((s) => s.precision !== null); + const avgPrecision = withPrecision.length + ? withPrecision.reduce((sum, s) => sum + (s.precision as number), 0) / withPrecision.length + : null; + return { total, passed, avgRecall, avgPrecision }; +} diff --git a/test/eval/fixtures/seed.ts b/test/eval/fixtures/seed.ts new file mode 100644 index 0000000..5f7c124 --- /dev/null +++ b/test/eval/fixtures/seed.ts @@ -0,0 +1,28 @@ +/** + * test/eval/fixtures/seed.ts - Seed a RecallScenario's sessions directly into + * a Smriti DB, bypassing real agent-log parsing entirely (mirrors the + * seedSession() helper in test/learn-consolidate.test.ts). + */ + +import type { Database } from "bun:sqlite"; +import { addMessage } from "../../../src/qmd"; +import { upsertProject, upsertSessionMeta, updateDensityScore } from "../../../src/db"; +import type { RecallScenario } from "./types"; + +const DAY_MS = 24 * 60 * 60 * 1000; + +export async function seedScenario(db: Database, scenario: RecallScenario): Promise { + const projects = new Set([scenario.project, ...scenario.sessions.map((s) => s.project ?? scenario.project)]); + for (const project of projects) upsertProject(db, project); + + for (const session of scenario.sessions) { + const createdAt = new Date(Date.now() - (session.daysAgo ?? 0) * DAY_MS).toISOString(); + for (const m of session.messages) { + await addMessage(db as any, session.id, m.role, m.content, { timestamp: createdAt }); + } + upsertSessionMeta(db, session.id, "claude-code", session.project ?? scenario.project); + if (session.densityScore !== undefined) { + updateDensityScore(db, session.id, session.densityScore); + } + } +} diff --git a/test/eval/fixtures/semantic-caching.ts b/test/eval/fixtures/semantic-caching.ts new file mode 100644 index 0000000..3dd4a6b --- /dev/null +++ b/test/eval/fixtures/semantic-caching.ts @@ -0,0 +1,40 @@ +import type { RecallScenario } from "./types"; + +/** + * Tier 2 only: the probe shares almost no lexical overlap with the source + * session (no "Redis", "cache", or "TTL" in the query) — only a real + * embedding model can bridge "in-memory data store to speed up repeated + * lookups" to "Redis... avoid repeated database round trips". + */ +export const SEMANTIC_CACHING: RecallScenario = { + name: "semantic-only match: caching decision, paraphrased query", + project: "api-service", + requiresEmbeddings: true, + sessions: [ + { + id: "semantic-s1", + messages: [ + { role: "user", content: "The product listing endpoint is slow under load." }, + { role: "assistant", content: "We store frequently accessed data in Redis to avoid repeated database round trips — added a 5-minute TTL for the product listing query." }, + ], + }, + { + id: "semantic-s2-distractor", + messages: [ + { role: "user", content: "Let's add rate limiting to the public API." }, + { role: "assistant", content: "Token bucket, 100 requests per minute per API key." }, + ], + }, + ], + probes: [ + { + query: "why do we use an in-memory data store to speed up repeated lookups", + description: "paraphrased, near-zero lexical overlap with the source session — needs semantic (vector) matching", + expectHitSessionIds: ["semantic-s1"], + expectMissSessionIds: ["semantic-s2-distractor"], + // The project-filtered path never touches vectors — route through + // recallMemories's hybrid pipeline, the only path embeddings affect. + useRecallMemories: true, + }, + ], +}; diff --git a/test/eval/fixtures/types.ts b/test/eval/fixtures/types.ts new file mode 100644 index 0000000..10fa61c --- /dev/null +++ b/test/eval/fixtures/types.ts @@ -0,0 +1,60 @@ +/** + * test/eval/fixtures/types.ts - Shared fixture format for the recall-quality + * harness. A scenario is a small multi-session conversation (optionally + * spanning sessions created at different points in time, via `daysAgo`) + * paired with probes that check both recall (the right session surfaces) + * and precision (a near-topic distractor does not). + * + * Shared by test/recall-quality.test.ts (Tier 1, CI-safe, BM25-only) and + * test/eval/recall-quality.eval.ts (Tier 2, manual, requires embeddings). + */ + +export type FixtureMessage = { role: "user" | "assistant"; content: string }; + +export type FixtureSession = { + id: string; + /** Overrides the scenario's default project — lets one scenario seed sessions across multiple projects to test project-filter isolation. */ + project?: string; + /** How many days before "now" this session was created — exercises recency/density blending. Omit for "just now". */ + daysAgo?: number; + /** density_score to set on this session (0-1). Omit to leave at the default (0). */ + densityScore?: number; + messages: FixtureMessage[]; +}; + +export type Probe = { + query: string; + /** Why a human would ask this — shown in the report, not asserted on. */ + description: string; + /** Session ids that MUST appear in the top-K results. */ + expectHitSessionIds: string[]; + /** At least one of these substrings must appear in a retrieved row belonging to an expected-hit session. */ + expectHitSubstrings?: string[]; + /** Session ids that must NOT appear in the top-K results (precision). */ + expectMissSessionIds?: string[]; + /** Defaults to the harness-wide DEFAULT_TOP_K. */ + topK?: number; + /** Overrides the scenario's default project for this probe's recall() call. */ + project?: string; + /** + * Route through recallMemories's unfiltered hybrid pipeline (RRF + density + * blending) instead of the project-filtered searchFiltered path. Still + * CI-safe without embeddings — vector search silently no-ops when none + * exist. Needed for probes that specifically exercise density/recency + * blending, which the project-filtered path never touches. + */ + useRecallMemories?: boolean; +}; + +export type RecallScenario = { + name: string; + /** Default project for sessions/probes that don't override it. */ + project: string; + sessions: FixtureSession[]; + probes: Probe[]; + /** + * Only runs in Tier 2 (manual, quality mode) — needs a live embedding + * backend to exercise genuinely non-lexical (semantic-only) matches. + */ + requiresEmbeddings?: boolean; +}; diff --git a/test/eval/recall-quality.eval.ts b/test/eval/recall-quality.eval.ts new file mode 100644 index 0000000..11c1196 --- /dev/null +++ b/test/eval/recall-quality.eval.ts @@ -0,0 +1,101 @@ +/** + * test/eval/recall-quality.eval.ts - Tier 2 of the recall-quality harness: + * the full fixture set from test/eval/fixtures/, including scenarios that + * need a live embedding backend to exercise genuinely semantic (non-lexical) + * matches — the CI-safe BM25-only subset already runs automatically as + * test/recall-quality.test.ts. + * + * NOT a bun:test file (no *.test.ts suffix) — needs a live embedding model + * (local llama.cpp, already a dependency, or Ollama via QMD_MEMORY_MODEL) and + * is slower than the CI subset, so it's excluded from `bun test` and run + * manually: + * + * bun run test/eval/recall-quality.eval.ts + */ + +import { initSmriti, closeDb } from "../../src/db"; +import { embedMemoryMessages } from "../../src/qmd"; +import { ALL_SCENARIOS } from "./fixtures/index"; +import { seedScenario } from "./fixtures/seed"; +import { runProbe, DEFAULT_TOP_K } from "./fixtures/run"; +import { summarizeScores, type ProbeScore } from "./fixtures/score"; +import type { RecallScenario } from "./fixtures/types"; + +type ProbeRun = { scenario: RecallScenario; query: string; description: string; score: ProbeScore; latencyMs: number; usedVectors: boolean }; + +async function runScenario(scenario: RecallScenario): Promise { + const db = await initSmriti(":memory:"); + const runs: ProbeRun[] = []; + try { + await seedScenario(db, scenario); + + let embedded = 0; + try { + embedded = await embedMemoryMessages(db as any); + } catch (err: any) { + console.log(` [warn] embedMemoryMessages failed for "${scenario.name}": ${err.message}`); + } + if (scenario.requiresEmbeddings && embedded === 0) { + console.log(` [warn] "${scenario.name}" needs embeddings but none were generated — results below may be BM25-only.`); + } + + for (const probe of scenario.probes) { + const { score, latencyMs, sources } = await runProbe(db, scenario, probe, { fast: false }); + runs.push({ + scenario, + query: probe.query, + description: probe.description, + score, + latencyMs, + usedVectors: sources.includes("vec"), + }); + } + } finally { + await closeDb(); + } + return runs; +} + +async function main() { + const embeddingScenarioCount = ALL_SCENARIOS.filter((s) => s.requiresEmbeddings).length; + console.log(`Running ${ALL_SCENARIOS.length} scenarios (${embeddingScenarioCount} require embeddings)...\n`); + + const allRuns: ProbeRun[] = []; + + for (const scenario of ALL_SCENARIOS) { + console.log(`## ${scenario.name}${scenario.requiresEmbeddings ? " (requires embeddings)" : ""}`); + const runs = await runScenario(scenario); + allRuns.push(...runs); + + for (const r of runs) { + const status = r.score.pass ? "PASS" : "FAIL"; + const vecTag = scenario.requiresEmbeddings ? (r.usedVectors ? " [vec]" : " [vec DID NOT FIRE]") : ""; + console.log( + ` [${status}] "${r.query}" — recall=${r.score.recall.toFixed(2)} precision=${r.score.precision === null ? "n/a" : r.score.precision.toFixed(2)} substrings=${r.score.substringOk}${vecTag} (${r.latencyMs.toFixed(0)}ms)` + ); + if (!r.score.pass) console.log(` ${r.description}`); + } + console.log(); + } + + const summary = summarizeScores(allRuns.map((r) => r.score)); + const vectorScenarios = allRuns.filter((r) => r.scenario.requiresEmbeddings); + const vectorsFired = vectorScenarios.filter((r) => r.usedVectors).length; + + console.log("=".repeat(60)); + console.log("SUMMARY"); + console.log("=".repeat(60)); + console.log(`Probes graded: ${summary.total}`); + console.log(`Probes passed: ${summary.passed}/${summary.total}`); + console.log(`Avg recall: ${summary.avgRecall.toFixed(2)}`); + console.log(`Avg precision: ${summary.avgPrecision === null ? "n/a" : summary.avgPrecision.toFixed(2)}`); + console.log(`Top-K: ${DEFAULT_TOP_K} (per-probe override via topK)`); + if (vectorScenarios.length > 0) { + console.log(`Vector search fired: ${vectorsFired}/${vectorScenarios.length} embedding-dependent probes`); + if (vectorsFired < vectorScenarios.length) { + console.log(` -> some embedding-dependent probes silently fell back to BM25-only. Check that a local embedding model or Ollama is reachable.`); + } + } +} + +await main(); diff --git a/test/eval/relation-inference.eval.ts b/test/eval/relation-inference.eval.ts new file mode 100644 index 0000000..010e576 --- /dev/null +++ b/test/eval/relation-inference.eval.ts @@ -0,0 +1,237 @@ +/** + * test/eval/relation-inference.eval.ts - Live A/B eval for relationship classification + * + * Compares classifyRelationshipsTextFormat ("before" — free-text RELATION + * lines parsed with a regex) against classifyRelationshipsToolCall ("after" + * — native tool calling) on a hand-labeled dataset, run against the real + * configured Ollama model (QMD_MEMORY_MODEL). + * + * NOT a bun:test file (no *.test.ts suffix) — it hits a live Ollama server + * and is slow/non-deterministic, so it's excluded from `bun test` and run + * manually: + * + * bun run test/eval/relation-inference.eval.ts + */ + +import { + classifyRelationshipsTextFormat, + classifyRelationshipsToolCall, + type RelationCandidate, + type RelationGuess, +} from "../../src/learn/consolidate"; + +// ============================================================================= +// Dataset +// ============================================================================= + +type Scenario = { + name: string; + unit: { topic: string; category: string; plainText: string }; + candidates: RelationCandidate[]; + expected: Array; // one expected predicate per candidate index +}; + +const SCENARIOS: Scenario[] = [ + { + name: "supersedes (reverted retrieval strategy)", + unit: { + topic: "Post-filtering for vector search", + category: "architecture/decision", + plainText: + "Switched the recall pipeline from pre-filtering to post-filtering with 3x overfetch, because pre-filtering caused sqlite-vec to hang when combined with JOINs on metadata tables.", + }, + candidates: [ + { id: "c1", topic: "Pre-filtering for vector search", category: "architecture/decision", plain_text: "Decision: use pre-filtering — apply metadata filters directly inside the sqlite-vec query before ranking." }, + ], + expected: ["supersedes"], + }, + { + name: "contradicts (daemon enrichment safety)", + unit: { + topic: "Inline LLM enrichment on daemon flush", + category: "architecture/decision", + plainText: "Team decided synchronous LLM enrichment on every daemon flush is safe and should run inline with ingestion.", + }, + candidates: [ + { id: "c1", topic: "Daemon flush safety", category: "architecture/decision", plain_text: "Decision: LLM enrichment must never run inline with daemon flush — it blocks ingestion and risks corrupting the write path under load." }, + ], + expected: ["contradicts"], + }, + { + name: "relatesTo (complementary recall features)", + unit: { + topic: "Cluster-scoped recall", + category: "feature/implementation", + plainText: "Added a --cluster flag to `smriti recall` for topic-scoped retrieval using O(1) Set membership checks.", + }, + candidates: [ + { id: "c1", topic: "RRF for recall", category: "feature/implementation", plain_text: "Implemented reciprocal rank fusion (RRF) to combine BM25 and vector search results in `smriti recall`." }, + ], + expected: ["relatesTo"], + }, + { + name: "none (unrelated domains)", + unit: { + topic: "Blog hover overlay CSS bug", + category: "bug/fix", + plainText: "Fixed a CSS bug where the hover overlay button showed a stray `.bv-tag` element on blog post cards.", + }, + candidates: [ + { id: "c1", topic: "Content hashing for dedup", category: "architecture/decision", plain_text: "Chose SHA256 content-addressable hashing for deduplicating ingested messages in QMD's content table." }, + ], + expected: ["none"], + }, + { + name: "supersedes (auth mechanism reversal)", + unit: { + topic: "Session cookies for auth", + category: "architecture/decision", + plainText: "Reverted from JWT-based session tokens to server-side session cookies after security review flagged JWT revocation as unsupported.", + }, + candidates: [ + { id: "c1", topic: "JWT session tokens", category: "architecture/decision", plain_text: "Adopted JWT-based session tokens for stateless auth across services." }, + ], + expected: ["supersedes"], + }, + { + name: "contradicts (MLX engine routing)", + unit: { + topic: "MLX engine routing in Ollama", + category: "topic/learning", + plainText: "Confirmed that `ollama pull` for MLX-tagged models requires model names ending in `-mlx`; regular GGUF pulls never use the MLX engine.", + }, + candidates: [ + { id: "c1", topic: "MLX engine routing in Ollama", category: "topic/learning", plain_text: "Established that any GGUF model pulled via `ollama pull` automatically runs on the MLX engine on Apple Silicon." }, + ], + expected: ["contradicts"], + }, + { + name: "relatesTo (ollama runner history)", + unit: { + topic: "Ollama runner architecture", + category: "topic/learning", + plainText: "Documented that Ollama's new llama-server-based runner replaced the old Go --ollama-engine runner starting in a later 0.x release.", + }, + candidates: [ + { id: "c1", topic: "Ollama runner architecture", category: "topic/learning", plain_text: "Verified Ollama 0.18 uses the ggml/Metal engine via a custom Go runner (--ollama-engine), not MLX, for standard GGUF models." }, + ], + expected: ["relatesTo"], + }, + { + name: "none (video upload vs tool-calling investigation)", + unit: { + topic: "Video metadata capture timeout", + category: "bug/fix", + plainText: "Fixed timeout and cancel handling in captureVideoMetadata to stop hangs during community photo/video upload.", + }, + candidates: [ + { id: "c1", topic: "MLX tool calling", category: "topic/learning", plain_text: "Investigated whether qwen3.5:9b-mlx-tuned supports native tool calling; confirmed via a /api/chat request with a get_weather tool schema." }, + ], + expected: ["none"], + }, + { + name: "multi-candidate index alignment", + unit: { + topic: "Standardize on post-filtering", + category: "architecture/decision", + plainText: "Standardized on post-filtering with 3x overfetch for all vector search filtering across projects.", + }, + candidates: [ + { id: "c1", topic: "Pre-filtering for vector search", category: "architecture/decision", plain_text: "Decision: use pre-filtering — apply metadata filters directly inside the sqlite-vec query." }, + { id: "c2", topic: "RRF for recall", category: "feature/implementation", plain_text: "Implemented RRF to merge BM25 and vector search scores." }, + { id: "c3", topic: "Blog hover overlay CSS bug", category: "bug/fix", plain_text: "Fixed a CSS bug in blog post hover overlays." }, + ], + expected: ["supersedes", "relatesTo", "none"], + }, +]; + +// ============================================================================= +// Runner +// ============================================================================= + +type MethodResult = { + guesses: RelationGuess[]; + latencyMs: number; + error?: string; +}; + +async function runMethod( + fn: (unit: Scenario["unit"], candidates: RelationCandidate[], model?: string) => Promise, + scenario: Scenario +): Promise { + const start = performance.now(); + try { + const guesses = await fn(scenario.unit, scenario.candidates); + return { guesses, latencyMs: performance.now() - start }; + } catch (err: any) { + return { guesses: [], latencyMs: performance.now() - start, error: err.message }; + } +} + +function grade(scenario: Scenario, guesses: RelationGuess[]) { + const byIndex = new Map(guesses.map((g) => [g.index, g.predicate])); + return scenario.expected.map((expected, index) => ({ + index, + expected, + got: byIndex.get(index) ?? "(missing)", + correct: byIndex.get(index) === expected, + })); +} + +async function main() { + console.log(`Running ${SCENARIOS.length} scenarios against both classifiers...\n`); + + let textCorrect = 0, toolCorrect = 0, total = 0; + let textParseFailures = 0, toolParseFailures = 0; + let textLatencyTotal = 0, toolLatencyTotal = 0; + + for (const scenario of SCENARIOS) { + console.log(`## ${scenario.name}`); + + // Sequential, not Promise.all: this Ollama server has OLLAMA_NUM_PARALLEL=1, + // so concurrent requests queue behind each other on the server anyway — + // running them "in parallel" here would just make one eat into the + // other's client-side timeout while it waits for a free slot. + const textResult = await runMethod(classifyRelationshipsTextFormat, scenario); + const toolResult = await runMethod(classifyRelationshipsToolCall, scenario); + + textLatencyTotal += textResult.latencyMs; + toolLatencyTotal += toolResult.latencyMs; + if (textResult.guesses.length === 0 && scenario.expected.length > 0) textParseFailures++; + if (toolResult.guesses.length === 0 && scenario.expected.length > 0) toolParseFailures++; + + const textGrades = grade(scenario, textResult.guesses); + const toolGrades = grade(scenario, toolResult.guesses); + + for (let i = 0; i < scenario.expected.length; i++) { + total++; + const t = textGrades[i]!; + const m = toolGrades[i]!; + if (t.correct) textCorrect++; + if (m.correct) toolCorrect++; + + console.log( + ` [${i}] expected=${t.expected.padEnd(11)} text-format=${String(t.got).padEnd(11)}${t.correct ? " ok " : " MISS"} tool-call=${String(m.got).padEnd(11)}${m.correct ? " ok " : " MISS"}` + ); + } + console.log( + ` latency: text-format=${textResult.latencyMs.toFixed(0)}ms tool-call=${toolResult.latencyMs.toFixed(0)}ms` + ); + if (textResult.error) console.log(` text-format error: ${textResult.error}`); + if (toolResult.error) console.log(` tool-call error: ${toolResult.error}`); + console.log(); + } + + console.log("=".repeat(60)); + console.log("SUMMARY"); + console.log("=".repeat(60)); + console.log(`Judgments graded: ${total}`); + console.log(`text-format accuracy: ${textCorrect}/${total} (${((textCorrect / total) * 100).toFixed(1)}%)`); + console.log(`tool-call accuracy: ${toolCorrect}/${total} (${((toolCorrect / total) * 100).toFixed(1)}%)`); + console.log(`text-format parse fails: ${textParseFailures}/${SCENARIOS.length} scenarios (zero guesses returned)`); + console.log(`tool-call parse fails: ${toolParseFailures}/${SCENARIOS.length} scenarios (zero guesses returned)`); + console.log(`text-format avg latency: ${(textLatencyTotal / SCENARIOS.length).toFixed(0)}ms`); + console.log(`tool-call avg latency: ${(toolLatencyTotal / SCENARIOS.length).toFixed(0)}ms`); +} + +await main(); diff --git a/test/forget.test.ts b/test/forget.test.ts new file mode 100644 index 0000000..6c38302 --- /dev/null +++ b/test/forget.test.ts @@ -0,0 +1,198 @@ +/** + * test/forget.test.ts - Tests for the smriti forget (session deletion) layer + * + * Mirrors test/learn-consolidate.test.ts's style: initSmriti(":memory:") so + * QMD's full schema (documents, content_vectors, vectors_vec) exists, a + * seedSession() helper for real message rows, and closeDb() teardown. + */ + +import { test, expect, beforeAll, afterAll } from "bun:test"; +import type { Database } from "bun:sqlite"; +import { + initSmriti, + closeDb, + upsertProject, + upsertSessionMeta, + insertKnowledgeUnit, + promoteKnowledgeUnit, + listKnowledgeUnits, + forgetSession, +} from "../src/db"; +import { listSessions } from "../src/qmd"; +import type { KnowledgeUnit } from "../src/team/types"; + +let db: Database; + +beforeAll(async () => { + db = await initSmriti(":memory:"); +}); + +afterAll(async () => { + await closeDb(); +}); + +function seedSession( + sessionId: string, + projectId: string, + messages: Array<{ role: string; content: string }> +) { + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO memory_sessions (id, title, created_at, updated_at) VALUES (?, ?, ?, ?)` + ).run(sessionId, `Session ${sessionId}`, now, now); + + const insertMsg = db.prepare( + `INSERT INTO memory_messages (session_id, role, content, hash, created_at) VALUES (?, ?, ?, ?, ?)` + ); + for (const [i, m] of messages.entries()) { + insertMsg.run(sessionId, m.role, m.content, `${sessionId}-h${i}`, now); + } + + upsertProject(db, projectId); + upsertSessionMeta(db, sessionId, "claude-code", projectId); +} + +const SAMPLE_MESSAGES = [ + { role: "user", content: "What's our rate limiting strategy?" }, + { role: "assistant", content: "Token bucket, 100 req/min per API key." }, +]; + +// ============================================================================= +// Soft delete +// ============================================================================= + +test("forgetSession soft-deletes by default: hidden from list, kept with includeInactive", () => { + seedSession("soft-s1", "forgetproj", SAMPLE_MESSAGES); + + const result = forgetSession(db, "soft-s1"); + expect(result.hard).toBe(false); + + const active = listSessions(db as any, { includeInactive: false }); + expect(active.map((s: any) => s.id)).not.toContain("soft-s1"); + + const all = listSessions(db as any, { includeInactive: true }); + expect(all.map((s: any) => s.id)).toContain("soft-s1"); + + // Messages are untouched by a soft delete. + const msgs = db + .prepare(`SELECT COUNT(*) as c FROM memory_messages WHERE session_id = ?`) + .get("soft-s1") as { c: number }; + expect(msgs.c).toBe(SAMPLE_MESSAGES.length); +}); + +// ============================================================================= +// Hard delete +// ============================================================================= + +test("forgetSession --hard removes messages, sidecar rows, and unpromoted knowledge units", () => { + seedSession("hard-s1", "forgetproj", SAMPLE_MESSAGES); + + db.prepare( + `INSERT INTO smriti_session_tags (session_id, category_id, confidence, source) VALUES (?, ?, ?, ?)` + ).run("hard-s1", "bug/fix", 0.9, "auto"); + + const segmented: KnowledgeUnit = { + id: "hard-unit-segmented", + topic: "Never promoted", + category: "code/pattern", + relevance: 2, + entities: [], + files: [], + plainText: "Low relevance, never promoted.", + lineRanges: [{ start: 0, end: 1 }], + }; + insertKnowledgeUnit(db, segmented, "hard-s1", "forgetproj", "hard-hash-1"); + db.prepare( + `INSERT INTO smriti_relationships (subject_type, subject_id, predicate, object_type, object_id) VALUES ('knowledge_unit', ?, 'mentions', 'entity', 'rate-limiting')` + ).run("hard-unit-segmented"); + + const result = forgetSession(db, "hard-s1", { hard: true }); + expect(result.hard).toBe(true); + expect(result.unitsDeleted).toBe(1); + + const msgs = db + .prepare(`SELECT COUNT(*) as c FROM memory_messages WHERE session_id = ?`) + .get("hard-s1") as { c: number }; + expect(msgs.c).toBe(0); + + const sessionRow = db.prepare(`SELECT 1 FROM memory_sessions WHERE id = ?`).get("hard-s1"); + expect(sessionRow).toBeNull(); + + const tags = db + .prepare(`SELECT COUNT(*) as c FROM smriti_session_tags WHERE session_id = ?`) + .get("hard-s1") as { c: number }; + expect(tags.c).toBe(0); + + const meta = db.prepare(`SELECT 1 FROM smriti_session_meta WHERE session_id = ?`).get("hard-s1"); + expect(meta).toBeNull(); + + const unit = db.prepare(`SELECT 1 FROM smriti_knowledge_units WHERE id = ?`).get("hard-unit-segmented"); + expect(unit).toBeNull(); + + const edges = db + .prepare(`SELECT COUNT(*) as c FROM smriti_relationships WHERE subject_id = ?`) + .get("hard-unit-segmented") as { c: number }; + expect(edges.c).toBe(0); +}); + +test("forgetSession --hard keeps canonical units and their doc/share unless --purge-shared", () => { + seedSession("hard-s2", "forgetproj", SAMPLE_MESSAGES); + + const canonical: KnowledgeUnit = { + id: "hard-unit-canonical", + topic: "Already shared decision", + category: "architecture/decision", + relevance: 9, + entities: [], + files: [], + plainText: "Already promoted and shared.", + lineRanges: [{ start: 0, end: 1 }], + }; + insertKnowledgeUnit(db, canonical, "hard-s2", "forgetproj", "hard-hash-2"); + promoteKnowledgeUnit(db, "hard-unit-canonical", "knowledge/architecture-decision/doc.md", "share-1"); + db.prepare( + `INSERT INTO smriti_shares (id, session_id, unit_id) VALUES (?, ?, ?)` + ).run("share-1", "hard-s2", "hard-unit-canonical"); + + const result = forgetSession(db, "hard-s2", { hard: true }); + expect(result.canonicalKept).toBe(1); + expect(result.unitsPurged).toBe(0); + + const kept = listKnowledgeUnits(db, { tier: "canonical" }).find( + (u) => u.id === "hard-unit-canonical" + ); + expect(kept).toBeDefined(); + + const share = db.prepare(`SELECT 1 FROM smriti_shares WHERE id = ?`).get("share-1"); + expect(share).toBeTruthy(); +}); + +test("forgetSession --hard --purge-shared removes canonical units and their share row", () => { + seedSession("hard-s3", "forgetproj", SAMPLE_MESSAGES); + + const canonical: KnowledgeUnit = { + id: "purge-unit-canonical", + topic: "Purge me too", + category: "architecture/decision", + relevance: 9, + entities: [], + files: [], + plainText: "Promoted, but this session is being fully purged.", + lineRanges: [{ start: 0, end: 1 }], + }; + insertKnowledgeUnit(db, canonical, "hard-s3", "forgetproj", "hard-hash-3"); + promoteKnowledgeUnit(db, "purge-unit-canonical", "knowledge/architecture-decision/purge-me.md", "share-2"); + db.prepare( + `INSERT INTO smriti_shares (id, session_id, unit_id) VALUES (?, ?, ?)` + ).run("share-2", "hard-s3", "purge-unit-canonical"); + + const result = forgetSession(db, "hard-s3", { hard: true, purgeShared: true }); + expect(result.unitsPurged).toBe(1); + expect(result.canonicalKept).toBe(0); + + const unit = db.prepare(`SELECT 1 FROM smriti_knowledge_units WHERE id = ?`).get("purge-unit-canonical"); + expect(unit).toBeNull(); + + const share = db.prepare(`SELECT 1 FROM smriti_shares WHERE id = ?`).get("share-2"); + expect(share).toBeNull(); +}); diff --git a/test/learn-consolidate.test.ts b/test/learn-consolidate.test.ts new file mode 100644 index 0000000..fa1a5bf --- /dev/null +++ b/test/learn-consolidate.test.ts @@ -0,0 +1,551 @@ +/** + * test/learn-consolidate.test.ts - Tests for continuous knowledge consolidation + * + * Mirrors test/team-segmented.test.ts's style: initSmriti(":memory:"), a + * mocked global.fetch standing in for Ollama, and a scratch tmpDir for + * filesystem output (never process.cwd() — consolidateKnowledge writes real + * files). + */ + +import { test, expect, beforeAll, afterAll, mock } from "bun:test"; +import type { Database } from "bun:sqlite"; +import { mkdirSync, rmSync, writeFileSync, existsSync, readFileSync } from "fs"; +import { join } from "path"; +import { tmpdir } from "os"; +import { + initSmriti, + closeDb, + upsertSessionMeta, + upsertProject, + updateDensityScore, + insertKnowledgeUnit, + listKnowledgeUnits, + promoteKnowledgeUnit, +} from "../src/db"; +import { + consolidateKnowledge, + pruneKnowledge, + classifyRelationshipsTextFormat, + classifyRelationshipsToolCall, +} from "../src/learn/consolidate"; +import { insertRelationship, getRelationships } from "../src/learn/entities"; +import { recall } from "../src/search/recall"; +import type { KnowledgeUnit } from "../src/team/types"; + +// ============================================================================= +// Setup +// ============================================================================= + +let db: Database; +let tmpDir: string; + +beforeAll(async () => { + db = await initSmriti(":memory:"); + tmpDir = join(tmpdir(), `smriti-consolidate-test-${Date.now()}`); + mkdirSync(tmpDir, { recursive: true }); +}); + +afterAll(async () => { + await closeDb(); + try { rmSync(tmpDir, { recursive: true }); } catch {} +}); + +/** Insert a session with real message rows so getSessionMessages() finds content. */ +function seedSession(sessionId: string, projectId: string, messages: Array<{ role: string; content: string }>) { + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO memory_sessions (id, title, created_at, updated_at) VALUES (?, ?, ?, ?)` + ).run(sessionId, `Session ${sessionId}`, now, now); + + const insertMsg = db.prepare( + `INSERT INTO memory_messages (session_id, role, content, hash, created_at) VALUES (?, ?, ?, ?, ?)` + ); + for (const [i, m] of messages.entries()) { + insertMsg.run(sessionId, m.role, m.content, `${sessionId}-h${i}`, now); + } + + upsertProject(db, projectId); + upsertSessionMeta(db, sessionId, "claude-code", projectId); +} + +const DENSE_CONVERSATION = [ + { role: "user", content: "I'm getting a JWT token expiry issue. Sessions timeout after 1 hour but tests expect 24 hours." }, + { role: "assistant", content: "Let me look at the auth middleware to understand the token expiry logic." }, + { role: "user", content: "Found it — src/auth.ts hardcodes 3600 seconds instead of reading JWT_TTL from the environment." }, + { role: "assistant", content: "Updated it to use process.env.JWT_TTL || 3600. Tests pass now." }, +]; + +/** Mock fetch that distinguishes Stage 1 (segmentation) vs Stage 2 (document) calls by prompt content. */ +function mockOllamaFetch(stage1Response: () => object) { + return mock(async (_url: string, init: any) => { + const body = JSON.parse(init.body); + const isStage1 = (body.prompt as string).includes("Knowledge Unit Segmentation"); + if (isStage1) { + return new Response( + JSON.stringify({ response: "```json\n" + JSON.stringify(stage1Response()) + "\n```" }), + { status: 200 } + ); + } + return new Response( + JSON.stringify({ response: "# Consolidated Doc\n\nPolished content." }), + { status: 200 } + ); + }); +} + +// ============================================================================= +// Segment-phase dedup +// ============================================================================= + +test("consolidate dedups knowledge units with identical content across sessions", async () => { + seedSession("dedup-s1", "dedupproj", DENSE_CONVERSATION); + seedSession("dedup-s2", "dedupproj", DENSE_CONVERSATION); + updateDensityScore(db, "dedup-s1", 0.9); + updateDensityScore(db, "dedup-s2", 0.9); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mockOllamaFetch(() => ({ + units: [{ topic: "JWT token expiry bug", category: "bug/fix", relevance: 9, entities: ["JWT"] }], + })) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 0.5, + outputDir: join(tmpDir, "dedup-output"), + }); + + expect(result.sessionsSegmented).toBe(2); + expect(result.unitsStored).toBe(1); + expect(result.unitsSkipped).toBe(1); + expect(result.errors).toEqual([]); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// Promotion threshold +// ============================================================================= + +test("promote phase only promotes units clearing the retrieval/relevance bar", async () => { + const belowThreshold: KnowledgeUnit = { + id: "unit-below", + topic: "Minor formatting note", + category: "code/pattern", + relevance: 3, + entities: [], + files: [], + plainText: "Use consistent indentation.", + lineRanges: [{ start: 0, end: 1 }], + }; + const aboveThreshold: KnowledgeUnit = { + id: "unit-above", + topic: "Redis caching decision", + category: "architecture/decision", + relevance: 9, + entities: ["Redis"], + files: [], + plainText: "Use Redis with a 5-minute TTL for API responses.", + lineRanges: [{ start: 0, end: 1 }], + }; + + insertKnowledgeUnit(db, belowThreshold, "promote-s1", "promoteproj", "hash-below"); + insertKnowledgeUnit(db, aboveThreshold, "promote-s2", "promoteproj", "hash-above"); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mockOllamaFetch(() => ({ units: [] })) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 999, // no sessions qualify for the segment phase — isolates promote phase + minRetrievals: 3, + minRelevance: 8, + outputDir: join(tmpDir, "promote-output"), + }); + + expect(result.unitsPromoted).toBe(1); + expect(result.errors).toEqual([]); + + const canonical = listKnowledgeUnits(db, { tier: "canonical" }); + expect(canonical.map((u) => u.id)).toContain("unit-above"); + expect(canonical.map((u) => u.id)).not.toContain("unit-below"); + + const promoted = canonical.find((u) => u.id === "unit-above")!; + expect(promoted.canonical_doc_path).toContain("architecture-decision"); + expect(promoted.share_id).toBeTruthy(); + + const shareRow = db + .prepare(`SELECT * FROM smriti_shares WHERE unit_id = ?`) + .get("unit-above") as any; + expect(shareRow).toBeTruthy(); + expect(shareRow.session_id).toBe("promote-s2"); + + const writtenFile = join(tmpDir, "promote-output", promoted.canonical_doc_path!); + expect(existsSync(writtenFile)).toBe(true); + + const stillSegmented = listKnowledgeUnits(db, { tier: "segmented" }); + expect(stillSegmented.map((u) => u.id)).toContain("unit-below"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// Graceful degradation +// ============================================================================= + +test("promote phase continues past a per-unit failure and records the error", async () => { + // Pre-create a *file* at the exact path the loop will try to mkdir for + // category "bug/fix" (slug "bug-fix"), forcing that unit's write to throw. + const outputDir = join(tmpDir, "degrade-output"); + const knowledgeDir = join(outputDir, "knowledge"); + mkdirSync(knowledgeDir, { recursive: true }); + writeFileSync(join(knowledgeDir, "bug-fix"), "occupied"); + + const willFail: KnowledgeUnit = { + id: "unit-fails", + topic: "Broken unit", + category: "bug/fix", + relevance: 9, + entities: [], + files: [], + plainText: "This unit's category dir collides with a file.", + lineRanges: [{ start: 0, end: 1 }], + }; + const willSucceed: KnowledgeUnit = { + id: "unit-succeeds", + topic: "Working unit", + category: "topic/learning", + relevance: 9, + entities: [], + files: [], + plainText: "This unit writes fine.", + lineRanges: [{ start: 0, end: 1 }], + }; + + insertKnowledgeUnit(db, willFail, "degrade-s1", "degradeproj", "hash-fails"); + insertKnowledgeUnit(db, willSucceed, "degrade-s2", "degradeproj", "hash-succeeds"); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mockOllamaFetch(() => ({ units: [] })) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 999, + minRetrievals: 999, // exclude leftover segmented units from earlier tests; only relevance-9 units below qualify + minRelevance: 8, + outputDir, + }); + + expect(result.unitsPromoted).toBe(1); + expect(result.errors.length).toBe(1); + expect(result.errors[0]).toContain("unit-fails"); + + const canonical = listKnowledgeUnits(db, { tier: "canonical" }); + expect(canonical.map((u) => u.id)).toContain("unit-succeeds"); + expect(canonical.map((u) => u.id)).not.toContain("unit-fails"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// Retrieval tracking (recall -> incrementRetrievalCount) +// ============================================================================= + +test("recall increments retrieval_count for knowledge units of the recalled session", async () => { + seedSession("track-s1", "trackproj", [ + { role: "user", content: "How do we configure the rate limiter for the public API?" }, + { role: "assistant", content: "Use a token bucket with 100 requests per minute per API key." }, + ]); + + const unit: KnowledgeUnit = { + id: "unit-tracked", + topic: "Rate limiter config", + category: "code/pattern", + relevance: 7, + entities: [], + files: [], + plainText: "Token bucket rate limiting for the public API.", + lineRanges: [{ start: 0, end: 1 }], + }; + insertKnowledgeUnit(db, unit, "track-s1", "trackproj", "hash-tracked"); + + await recall(db, "rate limiter", { project: "trackproj" }); + + const tracked = db + .prepare(`SELECT retrieval_count FROM smriti_knowledge_units WHERE id = ?`) + .get("unit-tracked") as { retrieval_count: number }; + expect(tracked).toBeDefined(); + expect(tracked.retrieval_count).toBeGreaterThanOrEqual(1); +}); + +test("recall does not throw for sessions with no consolidated knowledge units", async () => { + seedSession("untracked-s1", "untrackedproj", [ + { role: "user", content: "What's our deploy process look like?" }, + { role: "assistant", content: "Push to main triggers the CI pipeline and auto-deploys." }, + ]); + + await expect(recall(db, "deploy process", { project: "untrackedproj" })).resolves.toBeDefined(); +}); + +// ============================================================================= +// Relationship classification (text-format vs tool-call) +// ============================================================================= + +const RELATION_UNIT = { topic: "Post-filtering for vector search", category: "architecture/decision", plainText: "Switched to post-filtering." }; +const RELATION_CANDIDATES = [ + { id: "cand-0", topic: "Pre-filtering for vector search", category: "architecture/decision", plain_text: "Decided to pre-filter." }, + { id: "cand-1", topic: "Unrelated CSS fix", category: "bug/fix", plain_text: "Fixed a hover overlay." }, +]; + +test("classifyRelationshipsTextFormat parses RELATION lines even without literal brackets", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async () => + new Response(JSON.stringify({ response: "RELATION 0: supersedes\nRELATION [1]: none" }), { status: 200 }) + ) as any; + + try { + const guesses = await classifyRelationshipsTextFormat(RELATION_UNIT, RELATION_CANDIDATES, "test-model"); + expect(guesses).toEqual([ + { index: 0, predicate: "supersedes" }, + { index: 1, predicate: "none" }, + ]); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("classifyRelationshipsTextFormat returns nothing parseable when the model drifts off-format", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async () => + new Response(JSON.stringify({ response: "I think candidate 0 is superseded by the new unit." }), { status: 200 }) + ) as any; + + try { + const guesses = await classifyRelationshipsTextFormat(RELATION_UNIT, RELATION_CANDIDATES, "test-model"); + expect(guesses).toEqual([]); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("classifyRelationshipsToolCall parses structured tool_calls output", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async () => + new Response( + JSON.stringify({ + model: "test-model", + message: { + role: "assistant", + content: "", + tool_calls: [ + { + function: { + name: "record_relationships", + arguments: { + relationships: [ + { index: 0, predicate: "supersedes" }, + { index: 1, predicate: "none" }, + ], + }, + }, + }, + ], + }, + done: true, + }), + { status: 200 } + ) + ) as any; + + try { + const guesses = await classifyRelationshipsToolCall(RELATION_UNIT, RELATION_CANDIDATES, "test-model"); + expect(guesses).toEqual([ + { index: 0, predicate: "supersedes" }, + { index: 1, predicate: "none" }, + ]); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("classifyRelationshipsToolCall drops entries with out-of-range indices or invalid predicates", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async () => + new Response( + JSON.stringify({ + model: "test-model", + message: { + role: "assistant", + content: "", + tool_calls: [ + { + function: { + name: "record_relationships", + arguments: { + relationships: [ + { index: 0, predicate: "supersedes" }, + { index: 99, predicate: "relatesTo" }, + { index: 1, predicate: "maybe" }, + ], + }, + }, + }, + ], + }, + done: true, + }), + { status: 200 } + ) + ) as any; + + try { + const guesses = await classifyRelationshipsToolCall(RELATION_UNIT, RELATION_CANDIDATES, "test-model"); + expect(guesses).toEqual([{ index: 0, predicate: "supersedes" }]); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("classifyRelationshipsToolCall returns nothing when the model answers without calling the tool", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async () => + new Response( + JSON.stringify({ + model: "test-model", + message: { role: "assistant", content: "Candidate 0 looks superseded." }, + done: true, + }), + { status: 200 } + ) + ) as any; + + try { + const guesses = await classifyRelationshipsToolCall(RELATION_UNIT, RELATION_CANDIDATES, "test-model"); + expect(guesses).toEqual([]); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// Prune (stale segmented units, superseded canonical units) +// ============================================================================= + +function backdateUnit(unitId: string, daysAgo: number) { + db.prepare( + `UPDATE smriti_knowledge_units SET created_at = datetime('now', '-' || ? || ' days') WHERE id = ?` + ).run(daysAgo, unitId); +} + +test("pruneKnowledge dry-run reports stale segmented units without deleting them", async () => { + const stale: KnowledgeUnit = { + id: "prune-stale-1", + topic: "Never promoted, never retrieved", + category: "code/pattern", + relevance: 3, + entities: [], + files: [], + plainText: "Low relevance, sat unused for weeks.", + lineRanges: [{ start: 0, end: 1 }], + }; + insertKnowledgeUnit(db, stale, "prune-s1", "pruneproj", "prune-hash-1"); + backdateUnit("prune-stale-1", 45); + + const result = await pruneKnowledge(db, { dryRun: true, pruneStaleDays: 30, minRelevance: 8 }); + + expect(result.unitsPruned).toBe(0); + expect(result.unitsArchived).toBe(0); + expect(result.pruneCandidates?.map((c) => c.id)).toContain("prune-stale-1"); + + const stillThere = listKnowledgeUnits(db, { tier: "segmented" }).find((u) => u.id === "prune-stale-1"); + expect(stillThere).toBeDefined(); +}); + +test("pruneKnowledge --apply deletes stale segmented units and their relationship edges", async () => { + const stale: KnowledgeUnit = { + id: "prune-apply-1", + topic: "Deletable stale unit", + category: "code/pattern", + relevance: 2, + entities: [], + files: [], + plainText: "Stale and about to be deleted.", + lineRanges: [{ start: 0, end: 1 }], + }; + insertKnowledgeUnit(db, stale, "prune-s2", "pruneproj", "prune-hash-2"); + backdateUnit("prune-apply-1", 45); + insertRelationship(db, "knowledge_unit", "prune-apply-1", "mentions", "entity", "some-entity"); + + const result = await pruneKnowledge(db, { dryRun: false, pruneStaleDays: 30, minRelevance: 8 }); + + expect(result.unitsPruned).toBeGreaterThanOrEqual(1); + const deleted = db.prepare(`SELECT 1 FROM smriti_knowledge_units WHERE id = ?`).get("prune-apply-1"); + expect(deleted).toBeNull(); + + const edges = getRelationships(db, { subjectType: "knowledge_unit", subjectId: "prune-apply-1" }); + expect(edges.length).toBe(0); +}); + +test("pruneKnowledge never prunes high-relevance segmented units even at zero retrievals", async () => { + const highRelevance: KnowledgeUnit = { + id: "prune-high-relevance", + topic: "One consolidate run away from promoting", + category: "architecture/decision", + relevance: 9, + entities: [], + files: [], + plainText: "High relevance, just hasn't been promoted yet.", + lineRanges: [{ start: 0, end: 1 }], + }; + insertKnowledgeUnit(db, highRelevance, "prune-s3", "pruneproj", "prune-hash-3"); + backdateUnit("prune-high-relevance", 45); + + const result = await pruneKnowledge(db, { dryRun: true, pruneStaleDays: 30, minRelevance: 8 }); + + expect(result.pruneCandidates?.map((c) => c.id)).not.toContain("prune-high-relevance"); +}); + +test("pruneKnowledge archives superseded canonical units and appends a banner to their doc", async () => { + const outputDir = join(tmpDir, "prune-archive-output"); + const docRelPath = "knowledge/architecture-decision/old-doc.md"; + const docFullPath = join(outputDir, docRelPath); + mkdirSync(join(outputDir, "knowledge/architecture-decision"), { recursive: true }); + writeFileSync(docFullPath, "---\nid: old-unit\n---\n\n# Old guidance\n\nUse a 1-minute TTL."); + + const oldUnit: KnowledgeUnit = { + id: "prune-superseded", topic: "Old Redis TTL guidance", category: "architecture/decision", relevance: 9, + entities: [], files: [], plainText: "Use a 1-minute TTL.", lineRanges: [{ start: 0, end: 1 }], + }; + const newUnit: KnowledgeUnit = { + id: "prune-superseder", topic: "New Redis TTL guidance", category: "architecture/decision", relevance: 9, + entities: [], files: [], plainText: "Use a 5-minute TTL instead.", lineRanges: [{ start: 0, end: 1 }], + }; + insertKnowledgeUnit(db, oldUnit, "prune-s4", "pruneproj", "prune-hash-old"); + insertKnowledgeUnit(db, newUnit, "prune-s5", "pruneproj", "prune-hash-new"); + promoteKnowledgeUnit(db, "prune-superseded", docRelPath, "prune-share-1"); + promoteKnowledgeUnit(db, "prune-superseder", "knowledge/architecture-decision/new-doc.md", "prune-share-2"); + insertRelationship(db, "knowledge_unit", "prune-superseder", "supersedes", "knowledge_unit", "prune-superseded", { + source: "llm", + }); + + const result = await pruneKnowledge(db, { dryRun: false, outputDir }); + + expect(result.unitsArchived).toBe(1); + + const archived = listKnowledgeUnits(db, { tier: "archived" }).find((u) => u.id === "prune-superseded"); + expect(archived).toBeDefined(); + expect(archived!.archived_reason).toBe("superseded"); + + const docContent = readFileSync(docFullPath, "utf-8"); + expect(docContent).toContain("Archived"); + expect(docContent).toContain("New Redis TTL guidance"); + + // The supersedes edge that justified archiving (and any mentions edges) + // are the audit trail — left untouched, not cascade-deleted. + const supersedeEdge = getRelationships(db, { + subjectType: "knowledge_unit", subjectId: "prune-superseder", predicate: "supersedes", objectId: "prune-superseded", + }); + expect(supersedeEdge.length).toBe(1); +}); diff --git a/test/learn-entities-sync.test.ts b/test/learn-entities-sync.test.ts new file mode 100644 index 0000000..8b850c4 --- /dev/null +++ b/test/learn-entities-sync.test.ts @@ -0,0 +1,166 @@ +/** + * test/learn-entities-sync.test.ts - Team/org propagation of the entities + + * relationships layer via the existing share/sync git round-trip. + * + * This is the test that directly answers "how does this reach the org/team + * level": two independent in-memory DBs stand in for two teammates' machines, + * connected only through a shared tmp `.smriti/` directory (config.json + + * knowledge/*.md), exactly like real git-committed team knowledge. + */ + +import { test, expect, beforeAll, afterAll, mock } from "bun:test"; +import type { Database } from "bun:sqlite"; +import { mkdirSync, rmSync } from "fs"; +import { join } from "path"; +import { tmpdir } from "os"; +import { initSmriti, closeDb, insertKnowledgeUnit, upsertProject } from "../src/db"; +import { readConfig, writeConfig, exportEntities } from "../src/team/config"; +import { syncTeamKnowledge } from "../src/team/sync"; +import { consolidateKnowledge } from "../src/learn/consolidate"; +import { getEntity, resolveEntity, insertRelationship, getRelationships } from "../src/learn/entities"; +import type { KnowledgeUnit } from "../src/team/types"; + +// Two independent :memory: databases (via separate initSmriti calls) stand in +// for two teammates' machines. initSmriti's underlying store singleton is +// process-wide, but every call site below threads the returned `db` handle +// explicitly rather than going through the singleton getDb(), so the two +// stay genuinely isolated for everything this test touches. +let dbA: Database; +let dbB: Database; +let sharedDir: string; + +beforeAll(async () => { + dbA = await initSmriti(":memory:"); + dbB = await initSmriti(":memory:"); + sharedDir = join(tmpdir(), `smriti-propagation-test-${Date.now()}`); + mkdirSync(sharedDir, { recursive: true }); + + // Pre-existing FK requirement of syncTeamKnowledge/upsertSessionMeta, + // unrelated to the entities/relationships work: the importing machine + // must already have the project registered locally (smriti_session_meta + // FKs to smriti_projects). Both "teammates" in this test are in the same project. + upsertProject(dbA, "propproj"); + upsertProject(dbB, "propproj"); +}); + +afterAll(async () => { + await closeDb(); + try { rmSync(sharedDir, { recursive: true }); } catch {} +}); + +test("entity canonical id and unit-relationship edges converge across two machines via share -> sync", async () => { + // --- Machine A: create + promote a unit that mentions "Redis" --- + const unit: KnowledgeUnit = { + id: "prop-unit-a", + topic: "Redis TTL guidance", + category: "architecture/decision", + relevance: 9, + entities: ["Redis"], + files: [], + plainText: "Use a 5-minute TTL for API response caching.", + lineRanges: [], + }; + insertKnowledgeUnit(dbA, unit, "prop-session-a", "propproj", "prop-hash-a"); + const redisIdOnA = resolveEntity(dbA, "Redis")!; + insertRelationship(dbA, "knowledge_unit", "prop-unit-a", "mentions", "entity", redisIdOnA); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async () => + new Response(JSON.stringify({ response: "# Redis TTL Guidance\n\nUse a 5-minute TTL." }), { status: 200 }) + ) as any; + + try { + const result = await consolidateKnowledge(dbA, { + minDensity: 999, // no segment-phase sessions on A — isolates the promote phase + minRetrievals: 999, + minRelevance: 8, // unit's relevance (9) qualifies + outputDir: sharedDir, + }); + expect(result.unitsPromoted).toBe(1); + expect(result.errors).toEqual([]); + } finally { + globalThis.fetch = originalFetch; + } + + // --- Machine A "shares": publish its canonical entity registry to the shared config.json --- + // (this is the exact operation writeManifest performs in src/team/share.ts, just called + // directly here to isolate propagation from the rest of the share pipeline's session-querying) + const existingConfig = readConfig(sharedDir); + await writeConfig(sharedDir, { ...existingConfig, version: 2, entities: exportEntities(dbA) }); + + // --- Machine B has never seen "Redis" before --- + expect(getEntity(dbB, "redis")).toBeNull(); + + const syncResult = await syncTeamKnowledge(dbB, { inputDir: sharedDir }); + + expect(syncResult.errors).toEqual([]); + expect(syncResult.entitiesImported).toBeGreaterThanOrEqual(1); + expect(syncResult.imported).toBeGreaterThanOrEqual(1); + + // The entity converged onto the SAME canonical id — not an independently re-slugified duplicate. + const redisOnB = getEntity(dbB, "redis"); + expect(redisOnB).toBeTruthy(); + expect(redisOnB!.id).toBe(redisIdOnA); + expect(redisOnB!.aliases).toContain("Redis"); + + // The unit's "mentions" edge was re-created on B, referencing that same entity id. + const bMentions = getRelationships(dbB, { + subjectType: "knowledge_unit", + subjectId: "prop-unit-a", + predicate: "mentions", + objectType: "entity", + }); + expect(bMentions.map((r) => r.object_id)).toContain(redisIdOnA); +}); + +test("supersedes edges between shared units survive the round-trip with no canonicalization needed", async () => { + // Machine A: an existing unit, and a new one that supersedes it — both promoted. + const existing: KnowledgeUnit = { + id: "prop-superseded-unit", topic: "Old Redis TTL", category: "architecture/decision", relevance: 8, + entities: ["Redis"], files: [], plainText: "Use a 1-minute TTL.", lineRanges: [], + }; + const superseding: KnowledgeUnit = { + id: "prop-superseding-unit", topic: "New Redis TTL", category: "architecture/decision", relevance: 9, + entities: ["Redis"], files: [], plainText: "Use a 5-minute TTL instead.", lineRanges: [], + }; + insertKnowledgeUnit(dbA, existing, "prop-session-b1", "propproj", "prop-hash-existing"); + insertKnowledgeUnit(dbA, superseding, "prop-session-b2", "propproj", "prop-hash-superseding"); + const redisId = resolveEntity(dbA, "Redis")!; + insertRelationship(dbA, "knowledge_unit", "prop-superseded-unit", "mentions", "entity", redisId); + insertRelationship(dbA, "knowledge_unit", "prop-superseding-unit", "mentions", "entity", redisId); + // Simulate what promote-phase LLM relationship inference would have discovered: + insertRelationship(dbA, "knowledge_unit", "prop-superseding-unit", "supersedes", "knowledge_unit", "prop-superseded-unit", { source: "llm" }); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async () => + new Response(JSON.stringify({ response: "# Doc\n\nContent." }), { status: 200 }) + ) as any; + + try { + // Only "prop-superseding-unit" clears the bar this round (existing unit stays at relevance 8 + // < minRelevance 8.5, so we promote exactly the one whose frontmatter should carry the edge). + const result = await consolidateKnowledge(dbA, { + minDensity: 999, + minRetrievals: 999, + minRelevance: 8.5, + outputDir: sharedDir, + }); + expect(result.unitsPromoted).toBe(1); + } finally { + globalThis.fetch = originalFetch; + } + + await writeConfig(sharedDir, { ...readConfig(sharedDir), version: 2, entities: exportEntities(dbA) }); + + const syncResult = await syncTeamKnowledge(dbB, { inputDir: sharedDir }); + expect(syncResult.errors).toEqual([]); + + const bEdges = getRelationships(dbB, { + subjectType: "knowledge_unit", + subjectId: "prop-superseding-unit", + predicate: "supersedes", + }); + // No entity-style canonicalization needed here: unit ids are portable UUIDs, + // so the edge lands referencing the exact same object id on both machines. + expect(bEdges.map((r) => r.object_id)).toContain("prop-superseded-unit"); +}); diff --git a/test/learn-entities.test.ts b/test/learn-entities.test.ts new file mode 100644 index 0000000..90e65f8 --- /dev/null +++ b/test/learn-entities.test.ts @@ -0,0 +1,425 @@ +/** + * test/learn-entities.test.ts - Tests for canonical entity resolution and + * relationship triples (RDF-inspired knowledge graph layer). + * + * Mirrors test/learn-consolidate.test.ts's style: initSmriti(":memory:"), + * mocked global.fetch standing in for Ollama, scratch tmpDir for output. + */ + +import { test, expect, beforeAll, afterAll, mock } from "bun:test"; +import type { Database } from "bun:sqlite"; +import { mkdirSync, rmSync } from "fs"; +import { join } from "path"; +import { tmpdir } from "os"; +import { + initSmriti, + closeDb, + upsertSessionMeta, + upsertProject, + updateDensityScore, + insertKnowledgeUnit, + findPromotableUnits, +} from "../src/db"; +import { + resolveEntity, + getEntity, + findEntity, + insertRelationship, + getRelationships, + findRelatedCandidates, + getUnitsForEntity, +} from "../src/learn/entities"; +import { consolidateKnowledge } from "../src/learn/consolidate"; +import type { KnowledgeUnit } from "../src/team/types"; + +let db: Database; +let tmpDir: string; + +beforeAll(async () => { + db = await initSmriti(":memory:"); + tmpDir = join(tmpdir(), `smriti-entities-test-${Date.now()}`); + mkdirSync(tmpDir, { recursive: true }); +}); + +afterAll(async () => { + await closeDb(); + try { rmSync(tmpDir, { recursive: true }); } catch {} +}); + +function seedSession(sessionId: string, projectId: string, messages: Array<{ role: string; content: string }>) { + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO memory_sessions (id, title, created_at, updated_at) VALUES (?, ?, ?, ?)` + ).run(sessionId, `Session ${sessionId}`, now, now); + + const insertMsg = db.prepare( + `INSERT INTO memory_messages (session_id, role, content, hash, created_at) VALUES (?, ?, ?, ?, ?)` + ); + for (const [i, m] of messages.entries()) { + insertMsg.run(sessionId, m.role, m.content, `${sessionId}-h${i}`, now); + } + + upsertProject(db, projectId); + upsertSessionMeta(db, sessionId, "claude-code", projectId); +} + +type RelationGuessLite = { index: number; predicate: string }; + +/** + * Stage 1 (segmentSession) and Stage 2 (generateDocument) go through + * callOllama's /api/generate, with a `prompt` field on the request body. + * Promote-time relationship inference (classifyRelationshipsToolCall) goes + * through ollamaChat's /api/chat instead — no `prompt` field, a `messages` + * array plus a native tool call in the response instead of free text. + */ +function mockOllamaFetch(handlers: { stage1?: () => object; relation?: () => RelationGuessLite[]; stage2?: () => string }) { + return mock(async (_url: string, init: any) => { + const body = JSON.parse(init.body); + + if (typeof body.prompt === "string") { + if (body.prompt.includes("Knowledge Unit Segmentation")) { + return new Response( + JSON.stringify({ response: "```json\n" + JSON.stringify((handlers.stage1 ?? (() => ({ units: [] })))()) + "\n```" }), + { status: 200 } + ); + } + return new Response( + JSON.stringify({ response: (handlers.stage2 ?? (() => "# Doc\n\nContent."))() }), + { status: 200 } + ); + } + + return new Response( + JSON.stringify({ + model: "test-model", + message: { + role: "assistant", + content: "", + tool_calls: [ + { + function: { + name: "record_relationships", + arguments: { relationships: (handlers.relation ?? (() => []))() }, + }, + }, + ], + }, + done: true, + }), + { status: 200 } + ); + }); +} + +// ============================================================================= +// Entity resolution +// ============================================================================= + +test("resolveEntity merges exact-normalize variants (case/whitespace) onto one canonical id", () => { + const id1 = resolveEntity(db, "JWT")!; + const id2 = resolveEntity(db, " jwt ")!; + const id3 = resolveEntity(db, "JWT."); + + expect(id1).toBe(id2); + expect(id1).toBe(id3); + + const entity = getEntity(db, id1)!; + expect(entity.label).toBe("JWT"); // first-seen label wins + expect(entity.aliases).toContain("JWT"); + expect(entity.aliases).toContain("jwt"); + expect(entity.mention_count).toBe(3); +}); + +test("resolveEntity does not merge genuinely different wordings for the same concept", () => { + const jwtId = resolveEntity(db, "distinct-jwt-test")!; + const fullFormId = resolveEntity(db, "distinct-json-web-token-test")!; + + expect(jwtId).not.toBe(fullFormId); +}); + +test("resolveEntity returns null for blank labels", () => { + expect(resolveEntity(db, " ")).toBeNull(); +}); + +test("findEntity looks up by exact id or by label", () => { + resolveEntity(db, "Redis"); + expect(findEntity(db, "Redis")?.id).toBe("redis"); + expect(findEntity(db, "redis")?.id).toBe("redis"); + expect(findEntity(db, "nonexistent-entity-xyz")).toBeNull(); +}); + +// ============================================================================= +// Relationship triples +// ============================================================================= + +test("insertRelationship dedups via the UNIQUE constraint", () => { + insertRelationship(db, "knowledge_unit", "unit-a", "mentions", "entity", "redis"); + insertRelationship(db, "knowledge_unit", "unit-a", "mentions", "entity", "redis"); + + const rows = getRelationships(db, { subjectType: "knowledge_unit", subjectId: "unit-a", predicate: "mentions" }); + expect(rows.length).toBe(1); +}); + +test("getRelationships supports single-triple-pattern lookup", () => { + insertRelationship(db, "knowledge_unit", "unit-b", "supersedes", "knowledge_unit", "unit-a", { source: "llm" }); + + const bySubject = getRelationships(db, { subjectId: "unit-b" }); + expect(bySubject.some((r) => r.predicate === "supersedes" && r.object_id === "unit-a")).toBe(true); + + const byPredicate = getRelationships(db, { predicate: "supersedes" }); + expect(byPredicate.length).toBeGreaterThanOrEqual(1); +}); + +// ============================================================================= +// Segment phase: mentions edges created with no extra LLM calls +// ============================================================================= + +test("consolidate segment phase creates mentions edges for every stored entity, no extra LLM calls", async () => { + seedSession("ent-s1", "entproj", [ + { role: "user", content: "We need to decide on a caching strategy for the API. Considering Redis vs in-memory caching." }, + { role: "assistant", content: "Redis is better — it's external state, handles multi-instance, fast, and proven." }, + ]); + updateDensityScore(db, "ent-s1", 0.9); + + let fetchCallCount = 0; + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async (_url: string, init: any) => { + fetchCallCount++; + const body = JSON.parse(init.body); + const isStage1 = (body.prompt as string).includes("Knowledge Unit Segmentation"); + if (isStage1) { + // relevance kept low (1) deliberately: this unit stays in the shared + // test DB as tier='segmented' after this test, and must never satisfy + // a later test's minRelevance threshold (e.g. 8) via leftover state. + return new Response( + JSON.stringify({ + response: "```json\n" + JSON.stringify({ + units: [{ topic: "Redis caching decision", category: "architecture/decision", relevance: 1, entities: ["Redis", "Caching"] }], + }) + "\n```", + }), + { status: 200 } + ); + } + return new Response(JSON.stringify({ response: "unexpected call" }), { status: 200 }); + }) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 0.5, + minRetrievals: 999, + minRelevance: 999, // nothing promotable — isolates the segment phase + outputDir: join(tmpDir, "ent-output"), + }); + + expect(result.unitsStored).toBe(1); + expect(fetchCallCount).toBe(1); // segmentation only, no promote-time LLM calls + + const unitRow = db + .prepare(`SELECT id FROM smriti_knowledge_units WHERE session_id = 'ent-s1'`) + .get() as { id: string }; + + const mentions = getRelationships(db, { + subjectType: "knowledge_unit", + subjectId: unitRow.id, + predicate: "mentions", + }); + expect(mentions.length).toBe(2); + const objectIds = mentions.map((m) => m.object_id).sort(); + expect(objectIds).toEqual(["caching", "redis"]); + + const unitsForRedis = getUnitsForEntity(db, "redis"); + expect(unitsForRedis.map((u) => u.id)).toContain(unitRow.id); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// minEntityReach promotion criterion +// ============================================================================= + +test("minEntityReach promotes a unit whose entity is shared by >= K other units, even at 0 retrievals", () => { + const shared: KnowledgeUnit = { + id: "reach-unit-1", topic: "Topic A", category: "code/pattern", relevance: 2, + entities: [], files: [], plainText: "content A", lineRanges: [], + }; + const sharedOther1: KnowledgeUnit = { + id: "reach-unit-2", topic: "Topic B", category: "code/pattern", relevance: 2, + entities: [], files: [], plainText: "content B", lineRanges: [], + }; + const sharedOther2: KnowledgeUnit = { + id: "reach-unit-3", topic: "Topic C", category: "code/pattern", relevance: 2, + entities: [], files: [], plainText: "content C", lineRanges: [], + }; + + insertKnowledgeUnit(db, shared, "reach-s1", "reachproj", "reach-hash-1"); + insertKnowledgeUnit(db, sharedOther1, "reach-s2", "reachproj", "reach-hash-2"); + insertKnowledgeUnit(db, sharedOther2, "reach-s3", "reachproj", "reach-hash-3"); + + const entityId = resolveEntity(db, "shared-webhook-retries")!; + insertRelationship(db, "knowledge_unit", "reach-unit-1", "mentions", "entity", entityId); + insertRelationship(db, "knowledge_unit", "reach-unit-2", "mentions", "entity", entityId); + insertRelationship(db, "knowledge_unit", "reach-unit-3", "mentions", "entity", entityId); + + // Below scalar thresholds (relevance=2, retrieval_count=0) but 2 OTHER units share the entity. + const promotableWithReach = findPromotableUnits(db, 999, 999, 2); + expect(promotableWithReach.map((u) => u.id)).toContain("reach-unit-1"); + + // Without minEntityReach, the same unit is not promotable. + const promotableWithoutReach = findPromotableUnits(db, 999, 999); + expect(promotableWithoutReach.map((u) => u.id)).not.toContain("reach-unit-1"); +}); + +// ============================================================================= +// Promote-phase relationship inference (LLM-gated, bounded) +// ============================================================================= + +test("promote phase persists LLM-inferred relatesTo/supersedes/contradicts edges", async () => { + const existing: KnowledgeUnit = { + id: "infer-existing", topic: "Old Redis TTL guidance", category: "architecture/decision", relevance: 5, + entities: [], files: [], plainText: "Use a 1-minute TTL.", lineRanges: [], + }; + insertKnowledgeUnit(db, existing, "infer-s1", "inferproj", "infer-hash-existing"); + const redisId = resolveEntity(db, "infer-redis")!; + insertRelationship(db, "knowledge_unit", "infer-existing", "mentions", "entity", redisId); + + const promoted: KnowledgeUnit = { + id: "infer-new", topic: "New Redis TTL guidance", category: "architecture/decision", relevance: 9, + entities: ["infer-redis"], files: [], plainText: "Use a 5-minute TTL instead.", lineRanges: [], + }; + insertKnowledgeUnit(db, promoted, "infer-s2", "inferproj", "infer-hash-new"); + insertRelationship(db, "knowledge_unit", "infer-new", "mentions", "entity", redisId); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mockOllamaFetch({ relation: () => [{ index: 0, predicate: "supersedes" }] }) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 999, // no segment-phase sessions + minRetrievals: 999, + minRelevance: 8, // only infer-new (relevance 9) qualifies + outputDir: join(tmpDir, "infer-output"), + }); + + expect(result.unitsPromoted).toBe(1); + expect(result.errors).toEqual([]); + + const edges = getRelationships(db, { subjectType: "knowledge_unit", subjectId: "infer-new", predicate: "supersedes" }); + expect(edges.length).toBe(1); + expect(edges[0].object_id).toBe("infer-existing"); + expect(edges[0].source).toBe("llm"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("promote phase never asserts a directional predicate in both directions for the same pair", async () => { + // Two units sharing an entity, BOTH clearing the promotion bar in the same + // run — each independently asks "do I supersede the other" and (with a + // naive LLM/mock that doesn't reason about recency) can get "yes" from both + // sides. Only one direction should end up persisted. + const unitA: KnowledgeUnit = { + id: "bidir-unit-a", topic: "Redis TTL: 1 minute", category: "architecture/decision", relevance: 9, + entities: [], files: [], plainText: "Use a 1-minute TTL.", lineRanges: [], + }; + const unitB: KnowledgeUnit = { + id: "bidir-unit-b", topic: "Redis TTL: 15 minutes", category: "architecture/decision", relevance: 9, + entities: [], files: [], plainText: "Use a 15-minute TTL instead.", lineRanges: [], + }; + insertKnowledgeUnit(db, unitA, "bidir-s1", "bidirproj", "bidir-hash-a"); + insertKnowledgeUnit(db, unitB, "bidir-s2", "bidirproj", "bidir-hash-b"); + const entityId = resolveEntity(db, "bidir-redis")!; + insertRelationship(db, "knowledge_unit", "bidir-unit-a", "mentions", "entity", entityId); + insertRelationship(db, "knowledge_unit", "bidir-unit-b", "mentions", "entity", entityId); + + const originalFetch = globalThis.fetch; + // Always answers "supersedes" regardless of which side is asking — the + // worst case for this bug, and realistic for a small/local model given a + // prompt with no explicit recency signal. + globalThis.fetch = mockOllamaFetch({ relation: () => [{ index: 0, predicate: "supersedes" }] }) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 999, + minRetrievals: 999, + minRelevance: 8, // both unitA and unitB qualify + outputDir: join(tmpDir, "bidir-output"), + }); + + expect(result.unitsPromoted).toBe(2); + + const aToB = getRelationships(db, { + subjectType: "knowledge_unit", subjectId: "bidir-unit-a", predicate: "supersedes", objectId: "bidir-unit-b", + }); + const bToA = getRelationships(db, { + subjectType: "knowledge_unit", subjectId: "bidir-unit-b", predicate: "supersedes", objectId: "bidir-unit-a", + }); + // Exactly one direction persisted, never both. + expect(aToB.length + bToA.length).toBe(1); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("promote phase relationship inference is best-effort: a broken LLM response doesn't block promotion", async () => { + const existing: KnowledgeUnit = { + id: "badllm-existing", topic: "Existing", category: "code/pattern", relevance: 5, + entities: [], files: [], plainText: "content", lineRanges: [], + }; + insertKnowledgeUnit(db, existing, "badllm-s1", "badllmproj", "badllm-hash-existing"); + const entId = resolveEntity(db, "badllm-entity")!; + insertRelationship(db, "knowledge_unit", "badllm-existing", "mentions", "entity", entId); + + const promoted: KnowledgeUnit = { + id: "badllm-new", topic: "New", category: "code/pattern", relevance: 9, + entities: ["badllm-entity"], files: [], plainText: "content", lineRanges: [], + }; + insertKnowledgeUnit(db, promoted, "badllm-s2", "badllmproj", "badllm-hash-new"); + insertRelationship(db, "knowledge_unit", "badllm-new", "mentions", "entity", entId); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async (_url: string, init: any) => { + const body = JSON.parse(init.body); + const prompt = body.prompt as string; + if (prompt.includes("CANDIDATES")) throw new Error("connection refused"); + return new Response(JSON.stringify({ response: "# Doc\n\nContent." }), { status: 200 }); + }) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 999, + minRetrievals: 999, + minRelevance: 8, + outputDir: join(tmpDir, "badllm-output"), + }); + + // Promotion itself succeeds even though relationship inference failed. + expect(result.unitsPromoted).toBe(1); + expect(result.errors).toEqual([]); + + const edges = getRelationships(db, { subjectType: "knowledge_unit", subjectId: "badllm-new" }) + .filter((r) => r.predicate !== "mentions"); + expect(edges.length).toBe(0); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// findRelatedCandidates +// ============================================================================= + +test("findRelatedCandidates finds other units sharing a canonical entity, excluding self", () => { + const a: KnowledgeUnit = { id: "cand-a", topic: "A", category: "code/pattern", relevance: 5, entities: [], files: [], plainText: "a", lineRanges: [] }; + const b: KnowledgeUnit = { id: "cand-b", topic: "B", category: "code/pattern", relevance: 5, entities: [], files: [], plainText: "b", lineRanges: [] }; + insertKnowledgeUnit(db, a, "cand-s1", "candproj", "cand-hash-a"); + insertKnowledgeUnit(db, b, "cand-s2", "candproj", "cand-hash-b"); + + const sharedEntity = resolveEntity(db, "cand-shared-entity")!; + insertRelationship(db, "knowledge_unit", "cand-a", "mentions", "entity", sharedEntity); + insertRelationship(db, "knowledge_unit", "cand-b", "mentions", "entity", sharedEntity); + + const candidates = findRelatedCandidates(db, "cand-a", 5); + expect(candidates.map((c) => c.id)).toEqual(["cand-b"]); + expect(candidates.map((c) => c.id)).not.toContain("cand-a"); +}); diff --git a/test/recall-quality.test.ts b/test/recall-quality.test.ts new file mode 100644 index 0000000..99ccfa1 --- /dev/null +++ b/test/recall-quality.test.ts @@ -0,0 +1,38 @@ +/** + * test/recall-quality.test.ts - Tier 1 of the recall-quality harness: the + * BM25-only scenarios from test/eval/fixtures/, run deterministically with + * no embedding backend needed (recall's project-filtered path never touches + * vectors; recallMemories's vector search silently no-ops with none). + * + * This is the CI-safe subset — it catches regressions in recall()'s + * filtering/dedup/RRF/density-blending wiring automatically. The full + * fixture set (including embedding-dependent scenarios) runs manually via + * `bun run eval:recall` (test/eval/recall-quality.eval.ts). + */ + +import { test, expect } from "bun:test"; +import { initSmriti, closeDb } from "../src/db"; +import { CI_SCENARIOS } from "./eval/fixtures/index"; +import { seedScenario } from "./eval/fixtures/seed"; +import { runProbe } from "./eval/fixtures/run"; + +for (const scenario of CI_SCENARIOS) { + test(`recall quality: ${scenario.name}`, async () => { + const db = await initSmriti(":memory:"); + try { + await seedScenario(db, scenario); + + for (const probe of scenario.probes) { + const { score } = await runProbe(db, scenario, probe, { fast: true }); + + expect(score.recall, `recall for "${probe.query}" (${probe.description})`).toBe(1); + if (score.precision !== null) { + expect(score.precision, `precision for "${probe.query}" (${probe.description})`).toBe(1); + } + expect(score.substringOk, `substring check for "${probe.query}" (${probe.description})`).toBe(true); + } + } finally { + await closeDb(); + } + }); +}