diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9125d3b..fe75d8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: - name: Run tests # Fast PR validation on Linux only. - run: bun test test/ + run: bun run test test-merge: if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev') @@ -61,7 +61,7 @@ jobs: - name: Run tests # Full cross-platform test matrix for merge branches. - run: bun test test/ + run: bun run test dev-draft-release: name: Push(dev) / Draft Release diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 52f34e1..89f7f4d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,7 +32,7 @@ jobs: run: bun install - name: Run tests - run: bun test test/ + run: bun run test - name: Compute release metadata from conventional commits id: meta diff --git a/CLAUDE.md b/CLAUDE.md index 99f56dc..bc0d6d8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -134,8 +134,34 @@ smriti embed # Build vector embeddings smriti categorize # Auto-categorize sessions smriti share --project myapp # Export to .smriti/ for git smriti sync # Import team knowledge + +# Daemon (v0.8+) — cross-agent capture in the background +smriti daemon install # Install LaunchAgent/systemd unit; auto-start at login +smriti daemon status # PID, uptime, watched agents +smriti daemon stop # Graceful shutdown +smriti daemon logs # Tail the daemon log +smriti daemon uninstall # Reverse install +smriti daemon # Run in foreground (debugging) ``` +### The Claude Stop hook with the daemon + +When the daemon is installed, the Claude Stop hook becomes a 5ms socket +poke instead of a full ingest invocation. Recommended template: + +```bash +#!/bin/bash +SOCK="$HOME/.cache/smriti/daemon.sock" +if [ -S "$SOCK" ]; then + : | nc -U "$SOCK" 2>/dev/null +else + /usr/bin/lockf -t 0 /tmp/smriti-ingest.lock smriti ingest claude 2>/dev/null +fi +exit 0 +``` + +The `lockf` fallback keeps the system working when the daemon isn't running. + ## Project Structure ``` @@ -249,7 +275,7 @@ See `docs/internal/ingest-architecture.md` for details. | `COPILOT_STORAGE_DIR` | auto-detected per OS | VS Code workspaceStorage root override | | `SMRITI_PROJECTS_ROOT` | `~/zero8.dev` | Projects root for ID derivation | | `OLLAMA_HOST` | `http://127.0.0.1:11434` | Ollama endpoint | -| `QMD_MEMORY_MODEL` | `qwen3:8b-tuned` | Ollama model for synthesis | +| `QMD_MEMORY_MODEL` | `qwen3.5:9b-mlx-tuned` | Ollama model for synthesis (MLX engine)| | `SMRITI_CLASSIFY_THRESHOLD` | `0.5` | LLM classification trigger threshold | | `SMRITI_AUTHOR` | `$USER` | Git author for team sharing | | `SMRITI_DAEMON_DEBOUNCE_MS` | `30000` | Daemon file-stability wait (v0.4.0) | diff --git a/bun.lock b/bun.lock index f191cb2..174000a 100644 --- a/bun.lock +++ b/bun.lock @@ -5,6 +5,7 @@ "": { "name": "smriti", "dependencies": { + "fast-glob": "3.3.3", "node-llama-cpp": "^3.0.0", "picomatch": "^4.0.0", "qmd": "file:./qmd", @@ -51,6 +52,12 @@ "@node-llama-cpp/win-x64-vulkan": ["@node-llama-cpp/win-x64-vulkan@3.15.1", "", { "os": "win32", "cpu": "x64" }, "sha512-BPBjUEIkFTdcHSsQyblP0v/aPPypi6uqQIq27mo4A49CYjX22JDmk4ncdBLk6cru+UkvwEEe+F2RomjoMt32aQ=="], + "@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="], + + "@nodelib/fs.stat": ["@nodelib/fs.stat@2.0.5", "", {}, "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A=="], + + "@nodelib/fs.walk": ["@nodelib/fs.walk@1.2.8", "", { "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" } }, "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg=="], + "@octokit/app": ["@octokit/app@16.1.2", "", { "dependencies": { "@octokit/auth-app": "^8.1.2", "@octokit/auth-unauthenticated": "^7.0.3", "@octokit/core": "^7.0.6", "@octokit/oauth-app": "^8.0.3", "@octokit/plugin-paginate-rest": "^14.0.0", "@octokit/types": "^16.0.0", "@octokit/webhooks": "^14.0.0" } }, "sha512-8j7sEpUYVj18dxvh0KWj6W/l6uAiVRBl1JBDVRqH1VHKAO/G5eRVl4yEoYACjakWers1DjUkcCHyJNQK47JqyQ=="], "@octokit/auth-app": ["@octokit/auth-app@8.2.0", "", { "dependencies": { "@octokit/auth-oauth-app": "^9.0.3", "@octokit/auth-oauth-user": "^6.0.2", "@octokit/request": "^10.0.6", "@octokit/request-error": "^7.0.2", "@octokit/types": "^16.0.0", "toad-cache": "^3.7.0", "universal-github-app-jwt": "^2.2.0", "universal-user-agent": "^7.0.0" } }, "sha512-vVjdtQQwomrZ4V46B9LaCsxsySxGoHsyw6IYBov/TqJVROrlYdyNgw5q6tQbB7KZt53v1l1W53RiqTvpzL907g=="], @@ -155,6 +162,8 @@ "bottleneck": ["bottleneck@2.19.5", "", {}, "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw=="], + "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], + "bun-types": ["bun-types@1.3.8", "", { "dependencies": { "@types/node": "*" } }, "sha512-fL99nxdOWvV4LqjmC+8Q9kW3M4QTtTR1eePs94v5ctGqU8OeceWrSUaRw3JYb7tU3FkMIAjkueehrHPPPGKi5Q=="], "bytes": ["bytes@3.1.2", "", {}, "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="], @@ -251,12 +260,18 @@ "fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="], + "fast-glob": ["fast-glob@3.3.3", "", { "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", "glob-parent": "^5.1.2", "merge2": "^1.3.0", "micromatch": "^4.0.8" } }, "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg=="], + "fast-uri": ["fast-uri@3.1.0", "", {}, "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA=="], + "fastq": ["fastq@1.20.1", "", { "dependencies": { "reusify": "^1.0.4" } }, "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw=="], + "filename-reserved-regex": ["filename-reserved-regex@3.0.0", "", {}, "sha512-hn4cQfU6GOT/7cFHXBqeBg2TbrMBgdD0kcjLhvSQYYwm3s4B6cjvBfb7nBALJLAXqmU5xajSa7X2NnUud/VCdw=="], "filenamify": ["filenamify@6.0.0", "", { "dependencies": { "filename-reserved-regex": "^3.0.0" } }, "sha512-vqIlNogKeyD3yzrm0yhRMQg8hOVwYcYRfjEoODd49iCprMn4HL85gK3HcykQE53EPIpX3HcAbGA5ELQv216dAQ=="], + "fill-range": ["fill-range@7.1.1", "", { "dependencies": { "to-regex-range": "^5.0.1" } }, "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg=="], + "finalhandler": ["finalhandler@2.1.1", "", { "dependencies": { "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "on-finished": "^2.4.1", "parseurl": "^1.3.3", "statuses": "^2.0.1" } }, "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA=="], "follow-redirects": ["follow-redirects@1.15.11", "", {}, "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ=="], @@ -283,6 +298,8 @@ "get-proto": ["get-proto@1.0.1", "", { "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" } }, "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g=="], + "glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], + "gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="], "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="], @@ -313,10 +330,16 @@ "ipull": ["ipull@3.9.3", "", { "dependencies": { "@tinyhttp/content-disposition": "^2.2.0", "async-retry": "^1.3.3", "chalk": "^5.3.0", "ci-info": "^4.0.0", "cli-spinners": "^2.9.2", "commander": "^10.0.0", "eventemitter3": "^5.0.1", "filenamify": "^6.0.0", "fs-extra": "^11.1.1", "is-unicode-supported": "^2.0.0", "lifecycle-utils": "^2.0.1", "lodash.debounce": "^4.0.8", "lowdb": "^7.0.1", "pretty-bytes": "^6.1.0", "pretty-ms": "^8.0.0", "sleep-promise": "^9.1.0", "slice-ansi": "^7.1.0", "stdout-update": "^4.0.1", "strip-ansi": "^7.1.0" }, "optionalDependencies": { "@reflink/reflink": "^0.1.16" }, "bin": { "ipull": "dist/cli/cli.js" } }, "sha512-ZMkxaopfwKHwmEuGDYx7giNBdLxbHbRCWcQVA1D2eqE4crUguupfxej6s7UqbidYEwT69dkyumYkY8DPHIxF9g=="], + "is-extglob": ["is-extglob@2.1.1", "", {}, "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ=="], + "is-fullwidth-code-point": ["is-fullwidth-code-point@5.1.0", "", { "dependencies": { "get-east-asian-width": "^1.3.1" } }, "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ=="], + "is-glob": ["is-glob@4.0.3", "", { "dependencies": { "is-extglob": "^2.1.1" } }, "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg=="], + "is-interactive": ["is-interactive@2.0.0", "", {}, "sha512-qP1vozQRI+BMOPcjFzrjXuQvdak2pHNUMZoeG2eRbiSqyvbEf/wQtEOTOX1guk6E3t36RkaqiSt8A/6YElNxLQ=="], + "is-number": ["is-number@7.0.0", "", {}, "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng=="], + "is-promise": ["is-promise@4.0.0", "", {}, "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ=="], "is-unicode-supported": ["is-unicode-supported@2.1.0", "", {}, "sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ=="], @@ -347,6 +370,10 @@ "merge-descriptors": ["merge-descriptors@2.0.0", "", {}, "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g=="], + "merge2": ["merge2@1.4.1", "", {}, "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg=="], + + "micromatch": ["micromatch@4.0.8", "", { "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" } }, "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA=="], + "mime-db": ["mime-db@1.54.0", "", {}, "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ=="], "mime-types": ["mime-types@3.0.2", "", { "dependencies": { "mime-db": "^1.54.0" } }, "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A=="], @@ -415,6 +442,8 @@ "qs": ["qs@6.14.1", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ=="], + "queue-microtask": ["queue-microtask@1.2.3", "", {}, "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A=="], + "range-parser": ["range-parser@1.2.1", "", {}, "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="], "raw-body": ["raw-body@3.0.2", "", { "dependencies": { "bytes": "~3.1.2", "http-errors": "~2.0.1", "iconv-lite": "~0.7.0", "unpipe": "~1.0.0" } }, "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA=="], @@ -431,8 +460,12 @@ "retry": ["retry@0.13.1", "", {}, "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg=="], + "reusify": ["reusify@1.1.0", "", {}, "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw=="], + "router": ["router@2.2.0", "", { "dependencies": { "debug": "^4.4.0", "depd": "^2.0.0", "is-promise": "^4.0.0", "parseurl": "^1.3.3", "path-to-regexp": "^8.0.0" } }, "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ=="], + "run-parallel": ["run-parallel@1.2.0", "", { "dependencies": { "queue-microtask": "^1.2.2" } }, "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA=="], + "safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], @@ -497,6 +530,8 @@ "tar": ["tar@6.2.1", "", { "dependencies": { "chownr": "^2.0.0", "fs-minipass": "^2.0.0", "minipass": "^5.0.0", "minizlib": "^2.1.1", "mkdirp": "^1.0.3", "yallist": "^4.0.0" } }, "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A=="], + "to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="], + "toad-cache": ["toad-cache@3.7.0", "", {}, "sha512-/m8M+2BJUpoJdgAHoG+baCwBT+tf2VraSfkBgl0Y00qIWt41DJ8R5B8nsEw0I58YwF5IZH6z24/2TobDKnqSWw=="], "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="], @@ -567,6 +602,8 @@ "ipull/pretty-ms": ["pretty-ms@8.0.0", "", { "dependencies": { "parse-ms": "^3.0.0" } }, "sha512-ASJqOugUF1bbzI35STMBUpZqdfYKlJugy6JBziGi2EE+AL5JPJGSzvpeVXojxrr0ViUYoToUjb5kjSEGf7Y83Q=="], + "micromatch/picomatch": ["picomatch@2.3.2", "", {}, "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA=="], + "minizlib/minipass": ["minipass@3.3.6", "", { "dependencies": { "yallist": "^4.0.0" } }, "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw=="], "ora/log-symbols": ["log-symbols@6.0.0", "", { "dependencies": { "chalk": "^5.3.0", "is-unicode-supported": "^1.3.0" } }, "sha512-i24m8rpwhmPIS4zscNzK6MSEhk0DUWa/8iYQWxhffV8jkI4Phvs3F+quL5xvS0gdQR0FyTCMMH33Y78dDTzzIw=="], diff --git a/docs/internal/daemon-prd.md b/docs/internal/daemon-prd.md new file mode 100644 index 0000000..ca9dca4 --- /dev/null +++ b/docs/internal/daemon-prd.md @@ -0,0 +1,162 @@ +# Phase one: the Smriti daemon + +## How we ended up here + +Some weeks ago I noticed my laptop fan was loud. I ran `ps -ef`. There were forty-two `smriti ingest claude` processes running, the oldest from four days earlier. Collectively they had burned about nine CPU-days of work without anyone asking them to. + +That's the story this PRD comes out of. The full postmortem is in `docs/papers/stop-hook-never-stopped.md`, and a reflective companion in `docs/papers/only-by-staying.md`. Short version: the Stop hook on Claude Code ran `smriti ingest claude` after every response, with no locking, and on a long-running development setup the ingests stacked up faster than they finished. + +The first fix was six characters: `lockf -t 0`. It shipped within an hour and stopped the pile-up entirely. But it left an obvious next question — *why does the ingest take long enough for this to be a problem in the first place?* — and that question spiralled outward across an evening's design conversation into about four different daemon proposals, each more elaborate than the last. + +This PRD is what's left after that conversation. It's deliberately less than what the conversation produced, because most of what the conversation produced was *correct* in the abstract but *misaligned* with what Smriti is actually for. + +## What we thought the daemon was for + +The first draft of this PRD (now deleted) tried to make the daemon solve four things simultaneously: + +1. The hook pile-up problem (lockf already solved it; daemon would replace lockf). +2. Cold-start cost on every ingest (Bun runtime, SQLite open, eventually an in-process embedding model). +3. Routing `smriti search` and `smriti recall` through a warm socket for faster reads. +4. Coordinating with QMD's MCP daemon so the embedding model didn't load twice. + +That was a daemon designed to be the centrepiece of Smriti — a long-running process that owned everything important. It was also, in retrospect, the wrong shape for the actual product. + +Two arguments killed it. The first was about embedding: most of the cold-start cost lives in the embedding model (~2–5 seconds, ~300–500 MB), and that's not even an in-process concern today — QMD owns the embedding pipeline and runs it via `qmd embed` as a one-shot batch. The daemon would be "amortizing" a cost that didn't yet exist in our process. + +The second was about the north star. Smriti is a memory layer for engineering teams. The thing that matters is not how fast my own hook runs; it's whether a teammate's past mistake reaches me before I make the same one. None of the four jobs above touched that. + +What did matter was something I had under-valued: **capture across all the agents my team uses, not just Claude.** Today only Claude has hook-driven capture. Codex CLI, Cursor, Cline, Copilot — they all sit on disk, written to by their own log directories, and they only enter Smriti's index if I remember to run `smriti ingest ` manually. That's the actual reliability gap. A teammate using Cursor produces sessions Smriti can't see, because no agent-specific hook tells Smriti about them. + +A daemon that watches the filesystem and ingests as files arrive solves this. That's the daemon worth building. Not a faster hook; a universal one. + +## What we're actually building + +A long-running process called `smriti daemon`, started automatically at user login on macOS and Linux, whose job is to: + +- Watch all configured agent log directories (`~/.claude/projects/`, `~/.codex/`, `~/.cline/tasks/`, the VS Code workspaceStorage path for Copilot, and any custom dirs the user adds). +- React to filesystem events by scheduling a per-project ingest after a 30-second debounce window. +- Run the existing ingest pipeline — parser → resolver → store gateway → orchestrator — for the project that just had activity. No new ingest code; the daemon is a trigger, not a re-implementation. +- Accept a poke from the Claude Stop hook (over a Unix socket) as an *additional* signal, not the only one. The Claude hook becomes a hint that wakes the watcher; the watcher itself is authoritative. + +Everything else — `smriti search`, `smriti recall`, `smriti share`, `smriti embed` — keeps working exactly as it does today. The daemon writes into the same SQLite database that the CLI reads from. There is no read-side routing. There is no embedding-model handling. There is no backend transport, no auth, no dashboard. + +The daemon is one process. It exists per user, per machine. It does one job: keep the local Smriti index continuously up-to-date with whatever agent the user happens to be using. + +## What it looks like in daily use + +A new user installs Smriti via `brew install` (or whichever shipping mechanism we settle on). The install also writes a `~/Library/LaunchAgents/dev.zero8.smriti.plist` on macOS, or a `~/.config/systemd/user/smriti.service` on Linux, and registers it. From that moment on, every time the user logs in, the daemon starts. + +The user opens Cursor and works on a refactor for two hours. They never run a `smriti` command. The daemon is watching `~/Library/Application Support/Cursor/...` (or wherever Cursor writes its session logs); it sees files growing, waits 30 seconds for the writes to settle, then runs the Cursor ingest pipeline for that project. The session is in Smriti's index by the time the user types `smriti recall` for the first time that evening. + +If the user is also on Claude Code, the Stop hook still fires after each turn — but instead of spawning a fresh `bun /Users/.../smriti/src/index.ts ingest claude` (the original sin of nine CPU-days), the hook is now: + +```bash +#!/bin/bash +SOCK="$HOME/.cache/smriti/daemon.sock" +if [ -S "$SOCK" ]; then + : | nc -U "$SOCK" 2>/dev/null +else + /usr/bin/lockf -t 0 /tmp/smriti-ingest.lock smriti ingest claude 2>/dev/null +fi +exit 0 +``` + +A 5-millisecond socket poke when the daemon is running. The existing `lockf` fallback when it isn't. Either way, no pile-up is possible. + +If the user kills the daemon (intentionally or by reboot before login completes), the next time they log in it comes back. If they don't want the daemon at all, they can `smriti daemon uninstall` and the LaunchAgent / systemd unit is removed. + +## What survives, and what doesn't + +A few specific failure modes worth being honest about, because they shape what the daemon promises: + +**Crashes.** If the daemon crashes mid-ingest, the next filesystem event re-triggers the same project's ingest. The existing pipeline is incremental — `session-resolver.ts` tracks how many messages exist in the DB per session and only writes new ones. Duplicate-on-crash is safe. + +**Reboot during agent work.** If the machine reboots while Claude is in the middle of writing a session, the daemon comes up at login and the watcher picks up the in-progress files on the next FS event. There's no in-flight state in the daemon worth preserving across reboots. + +**Manual ingest still works.** `smriti ingest claude` from the CLI continues to do exactly what it does today. It's a perfectly valid fallback when the daemon isn't running, for users who don't want a daemon, or for testing. The daemon doesn't replace it; it just makes it usually-unnecessary. + +**`smriti share` still works, unchanged.** The existing sanitization in `src/team/formatter.ts` continues to handle the basic cleanup. We are explicitly *not* adding a real redaction pipeline in this phase. That comes later, when transport to a backend is on the table. + +**Cursor / Copilot capture is best-effort.** We don't control the formats these tools use; if they change layouts, the watcher might miss files until we update the discovery code. This is the same robustness story as today's manual ingest — the daemon doesn't make it worse, it just makes it more visible because there's nothing else to blame. + +## What we're explicitly not building + +Every item below came up in the design conversation, and every one of them is a separate phase. Enumerating them here so the boundary of phase one is clear: + +- **No backend service, no transport, no auth.** The daemon writes to the local SQLite. Nothing leaves the machine in this phase. If you want team sharing, you still use `smriti share` exactly as today: commits curated knowledge to `.smriti/`, pushes via git. +- **No real redaction pipeline.** Sanitization stays at its current level. The day we add transport, redaction becomes the next phase's gating concern. Until then, the existing share behaviour is fine. +- **No read-side routing.** `smriti search` and `smriti recall` continue to be one-shot CLI invocations that open SQLite and run a query. They cold-start in ~150ms; that's not great, but it's not bad enough to justify the lifetime complexity of routing reads through a socket. +- **No coordination with QMD's MCP daemon.** The Smriti daemon doesn't know or care whether `qmd mcp --daemon` is running. They share the same SQLite file via WAL, which handles concurrent writers and readers correctly. If both daemons run at once, each opens its own SQLite handle; nothing breaks. +- **No QMD upstream proposal.** The `searchFTS({ joins })` idea is good and still on my list, but it's orthogonal to the daemon and shouldn't gate this work. +- **No Windows.** Bun on Windows is rough, named pipes have their own quirks, and we don't have a Windows user we care about. macOS and Linux only. +- **No raw-transcript-to-git pipeline (Entire-style).** That belongs to whatever the eventual team-sharing architecture is. Today's curated `smriti share` is sufficient. +- **No embedding model in the daemon.** The daemon does parse-and-write; embeddings are still computed by `qmd embed` (manual or scheduled). When vector staleness becomes the user-visible problem, we'll revisit. + +The product of saying "no" to all of the above is that **phase one is implementable in roughly a week**, not a month. That matters more than getting any one of those right pre-emptively. + +## How the code lays out + +A new directory under `src/`: + +``` +src/daemon/ +├── server.ts // PID-file single-instance guard, IPC socket, signal handling +├── watcher.ts // native fs.watch (macOS recursive); walk-and-watch fallback (Linux) +├── queue.ts // per-project debounce + ingest dispatch +├── client.ts // smriti daemon stop/status helpers +├── install.ts // generate + register LaunchAgent / systemd unit +└── handlers.ts // poke handler (Claude Stop hook) +``` + +### Three pre-impl smoke-test findings that shaped these choices + +Each of these was verified before any production code was written, against Bun 1.3.6 on macOS. They each turned an instinct from the design conversation into a different choice in the implementation. + +**Single-instance via PID file, not socket-bind.** The obvious first instinct was to bind a Unix socket and rely on `EADDRINUSE` to detect a second daemon. Under Bun this is silently broken: `net.createServer().listen(path)` succeeds on the second call and *steals* incoming connections from the first server with no error. The first server thinks it's still listening but receives nothing. The test was concrete — start two servers in one Bun process, send three connections, find s1 got 0 and s2 got 3. + +So single-instance is enforced via QMD's pattern instead: a PID file at `~/.cache/smriti/daemon.pid` plus a `process.kill(pid, 0)` liveness probe on startup. The Unix socket continues to be used for IPC (the hook poke), but not as the guard. Startup order is: check PID file → exit if a live daemon owns it → write our PID → bind the IPC socket → start watching. + +**Native `fs.watch`, not chokidar.** Initial instinct was to lean on `chokidar` for cross-platform watching. Test: chokidar 5.0.0 watching `~/.claude/projects/` under Bun, with two self-touched files inside the window. Events seen: zero. The watcher reaches `ready` but never fires. Replacing chokidar with Node's bare `fs.watch(root, { recursive: true })` produced four events on the same workload — including, satisfyingly, the live JSONL writes from the Claude session that was running the test. + +So `src/daemon/watcher.ts` uses native `fs.watch` directly. macOS gets recursive watching for free; Linux needs a walk-and-watch fallback (Linux `fs.watch` doesn't implement `recursive`), which we'll hand-roll rather than pull in a watching library. + +**Open the DB connection per ingest cycle, not per daemon lifetime.** The original instinct was that the daemon would open SQLite once at startup and reuse the connection across every ingest flush. The test for that — call `ingest()` five times in a single Bun process — exposed three problems. RSS climbed to 6.8 GB peak. The process retained roughly 20 extra file descriptors. And Bun itself panicked partway through with a segfault at `0xC9AB8`, repeatably. + +Whatever lives downstream of `ingest()` is not safe to reuse across iterations under today's Bun + QMD versions. So the daemon does the boring thing: opens a fresh SQLite handle at the start of each per-project debounce flush, runs `ingest()`, closes the handle. The cost is ~20–50ms of cold-open time per cycle, which is invisible inside a 30-second debounce window. The benefit is that we sidestep the leak entirely. The real fix lives upstream (track down the FD/RSS accumulation in QMD's store layer, file the Bun crash with a minimal repro) but neither blocks shipping. + +The existing ingest pipeline (`src/ingest/`) is reused as a library — the daemon imports the orchestrator from `src/ingest/index.ts` and runs it in-process. No subprocess spawning, no CLI invocation, no extra Bun cold start per fire. + +New CLI subcommands: + +- `smriti daemon` — run in foreground (debugging, systemd target) +- `smriti daemon install` — write the LaunchAgent / systemd unit file, register it, start it +- `smriti daemon uninstall` — reverse of install; daemon stops and the unit file is removed +- `smriti daemon status` — PID, uptime, pending queues, last ingest per project +- `smriti daemon stop` — graceful shutdown via socket; fallback to PID-file SIGTERM +- `smriti daemon logs` — tail the rotating log at `~/.cache/smriti/daemon.log` + +The CLI keeps working without the daemon. None of the existing commands grow a daemon dependency. + +## How we'll know it worked + +The smallest set of criteria that distinguishes *shipped correctly* from *shipped but broken*: + +1. After `smriti daemon install`, the daemon survives a logout/login cycle and a full reboot without user intervention. +2. Opening Cursor on a new project, doing some work, and *never running a smriti command* — that project's sessions appear in `smriti search` within `30s + ingest_time` of the work being saved. +3. Claude Code's Stop hook completes within 50 milliseconds when the daemon is running. +4. SIGKILLing the daemon leaves no stale socket, no stale PID file, no corrupt SQLite state. Re-running it works cleanly. +5. Running `smriti daemon install` twice produces idempotent results — same plist/service file, same registered job, no duplicates. +6. `smriti share` continues to work, unchanged, with its existing sanitization. No new redaction error paths. +7. Removing the daemon (`smriti daemon uninstall`) leaves the system in exactly the state it was before installation — no orphaned files, no lingering processes. + +## What comes after + +This is the first phase. The next is a real redaction pipeline — high-entropy detection, credentialed URI scrubbing, vendor secret patterns, typed placeholders — that re-shapes `smriti share` to handle raw transcripts safely alongside the curated knowledge it already produces. That work becomes load-bearing the moment Smriti starts handling transcripts at any kind of scale. + +Beyond that, the trajectory tracks what users actually ask for: additional agent integrations, search quality improvements as QMD evolves, ergonomics around the team-sharing flow. Phase one alone produces a meaningfully better Smriti for anyone using more than one coding agent. + +## Closing the loop + +The daemon we're building is much smaller than the daemon we started designing. That's deliberate — most of what we initially put in it was solving for problems Smriti doesn't yet have, or problems that belong to other parts of the system. The version that ships is the version that does exactly one new thing well: capture across all my agents, automatically, so I never have to think about which one I used yesterday. + +Everything else is on the runway, in order. Phase one first. diff --git a/docs/internal/release-flow.md b/docs/internal/release-flow.md new file mode 100644 index 0000000..c10f514 --- /dev/null +++ b/docs/internal/release-flow.md @@ -0,0 +1,134 @@ +# Release flow + +How a Smriti version goes from "code on a feature branch" to "tagged release that downstream users pick up via `smriti upgrade`." Written after the v0.8.0 work; intended to be reused for every future release. + +## Versioning + +Standard semver. The general rule: features bump minor (0.7.0 → 0.8.0), bug fixes and polish bump patch (0.8.0 → 0.8.1), and we will reach 1.0.0 once the daemon has been running on real teams for a month without anyone hitting a "this is broken in a load-bearing way" issue. + +A note about local drift: `package.json` has occasionally lagged the git tag (v0.7.0 was tagged without a corresponding `"version"` bump). Try to keep them in sync; if they drift, fix it during the next release rather than rewriting history. + +## The four phases + +### Phase 1 — Feature branch development + +All work for a release lives on one feature branch, named `feat/` (e.g. `feat/daemon-core` for v0.8.0). One feature branch per release, even if the release contains several modules. + +The branch accumulates commits as we go. We don't try to keep the branch always-rebased-to-main during development — that creates more friction than it solves for a solo developer. Instead we squash-or-merge when we're ready to ship. + +Each commit on the branch should be independently testable: `bun test` passes after each commit. This makes bisecting later much cheaper. + +### Phase 2 — Staging on real hardware + +When the branch is feature-complete and unit-tested, we install it on real hardware and exercise it. There's no separate "build artifact" — the source IS the build, and switching to staging is `git checkout && bun install`. + +For the developer (running from `/Users/zero8/zero8.dev/smriti/`): +```bash +cd /Users/zero8/zero8.dev/smriti +git fetch origin +git checkout feat/ +bun install --frozen-lockfile +bun test # sanity +``` + +For downstream users (running from `~/.smriti` as a git clone): +```bash +cd ~/.smriti +git fetch origin +git checkout feat/ +bun install --frozen-lockfile +``` + +If the release adds a long-running process or service-file install, the staging step includes those too — e.g. for v0.8.0: +```bash +bun src/index.ts daemon install +bun src/index.ts daemon status # verify it's running +``` + +To leave staging, `git checkout main && bun install --frozen-lockfile` (and remove any service-file installs). + +### Phase 3 — Release-readiness checklist + +Every release has a tracking issue with a checklist of real-hardware verifications. The checklist is release-specific (a daemon release tests reboot + soak; a search-quality release tests recall against a fixture set; etc.) but the shape is consistent: + +- Per-OS verification rows that have to be done on actual machines +- Soak / endurance rows where time itself is the test +- Idempotency rows (running install twice, etc.) that catch state-corruption bugs +- A "previous CLI still works" row to catch regressions + +When all rows are ✅, we tag. When a row fails, we fix it on the feature branch with a small commit and re-run the row — same branch, just more commits. + +The tracking issue for v0.8.0 is #75. Future releases should clone its structure. + +### Phase 4 — Promotion to release + +```bash +# 1. Final sanity check +cd /Users/zero8/zero8.dev/smriti +git checkout feat/ +bun test # all green +bun src/index.ts # smoke-test the headline feature + +# 2. Merge the PR +gh pr merge --squash --delete-branch # squash if many commits and you don't need the history + # --merge if you want the commit-by-commit story preserved + +# 3. Tag +git checkout main && git pull +git tag -a v -m "v — " +git push origin v + +# 4. GitHub release with notes +gh release create v \ + --title "v — " \ + --notes-file docs/internal/release-notes-v.md \ + --latest +``` + +The release notes file lives in the repo (`docs/internal/release-notes-v.md`) as a draft from Phase 1, gets polished during Phase 3, and is the canonical source for the GitHub release body in Phase 4. After tagging, the file can stay in the repo as historical record — it's small and useful when someone asks "what landed in 0.8?" + +### Optional Phase 5 — Daemon / long-running-process restart + +For releases that ship changes to a long-running process (the daemon, future MCP server, etc.), users who upgrade need to restart that process to pick up the new code. Today this is manual: + +```bash +smriti upgrade # git pull + bun install +smriti daemon stop +launchctl kickstart -k gui/$UID/dev.zero8.smriti # macOS; KeepAlive=true will respawn it +# or: systemctl --user restart smriti # Linux +``` + +A v0.8.1 polish release should teach `smriti upgrade` to detect a running daemon and restart it automatically. Tracked separately — not load-bearing for v0.8.0 itself. + +## What lives where + +| Artifact | Location | When updated | +|---|---|---| +| Release-tracking issue | GitHub issue (one per release) | Created at start of Phase 3; closed when tagged | +| Release notes | `docs/internal/release-notes-v.md` | Drafted Phase 1, polished Phase 3, used in Phase 4 | +| Version | `package.json` `"version"` | Bumped in the same commit as the release notes finalisation | +| CHANGELOG | We don't maintain one. The set of GitHub Releases is the changelog. | — | +| Reference doc per major change | `docs/internal/*-prd.md` | Drafted alongside the feature; stays in the repo as historical record | +| Postmortems / reflections | `docs/papers/` | When something is worth telling as a story | + +## What we deliberately don't do + +- **No CI release pipeline.** Releases are small enough and rare enough that automating them past `gh release create` adds more failure modes than it removes. If we ever release multiple times a week, revisit. +- **No release candidates or beta channels.** Staging on the feature branch IS the RC. If a release needs longer soak time before tagging, just leave it in Phase 3 longer. +- **No release branches.** `main` is always the latest stable; feature branches are everything else. Branching off a tag for a hotfix is fine, but we don't keep a `release/0.8.x` branch alive after tagging. +- **No version-skipping for ceremony.** If v0.7.0 was tagged without a `package.json` bump, the next release just skips ahead in `package.json` — we don't go back and re-tag 0.7.1 to fix the drift. + +## When something goes wrong post-release + +If a release ships with a regression bad enough to revert: + +1. `git revert ` on main (creates a clean revert commit) +2. Tag v from the revert +3. Push tag, create release marked as a regression revert +4. Users on `smriti upgrade` pick up the revert via the normal flow + +For less severe issues, a regular patch release (v with the fix) is preferred over a revert. + +## Source of truth for the current release + +Always the GitHub release for the highest tag. If `package.json` disagrees with the tag, the tag wins. If a doc disagrees with the code, the code wins. We are explicit about this so future-us doesn't get confused by stale documentation that says we shipped something we didn't. diff --git a/docs/internal/release-notes-v0.8.0.md b/docs/internal/release-notes-v0.8.0.md new file mode 100644 index 0000000..dc2cbec --- /dev/null +++ b/docs/internal/release-notes-v0.8.0.md @@ -0,0 +1,95 @@ +# Smriti v0.8.0 — Cross-agent capture, finally + +The headline: a long-running `smriti daemon` that captures your sessions across every coding agent in the background. Open Cursor, Codex, or Claude — the daemon watches the filesystem, debounces, and ingests automatically. You stop having to remember which agent you used yesterday, or whether you remembered to `smriti ingest`. + +This is the release the postmortem [`docs/papers/stop-hook-never-stopped.md`](https://github.com/zero8dotdev/smriti/blob/main/docs/papers/stop-hook-never-stopped.md) gestured at — the daemon shape the lockf mitigation pointed toward. It also reuses everything Smriti was already doing for Claude (the Stop hook continues to work, just as a 5ms socket poke now instead of a full ingest). + +## What you get + +- **Cross-agent capture.** Sessions from Claude, Codex, Cline, Copilot, and Cursor are picked up automatically as they're written. No `smriti ingest ` to remember. +- **Auto-start at login.** `smriti daemon install` writes a LaunchAgent on macOS or a systemd-user unit on Linux. Daemon comes back after every reboot; restarts itself on crash. +- **Per-project debouncing.** A busy session in project A doesn't delay project B. Each project gets its own 30s settle window. +- **Six new commands** (`smriti daemon install / uninstall / status / stop / logs`, plus the bare `smriti daemon` for foreground debugging). +- **`smriti share` continues to work** with its existing sanitization — unchanged. + +## Recommended Claude Stop-hook update + +When the daemon is running, the Stop hook becomes a 5ms poke. Update your `~/.claude/hooks/save-memory.sh` to: + +```bash +#!/bin/bash +SOCK="$HOME/.cache/smriti/daemon.sock" +if [ -S "$SOCK" ]; then + : | nc -U "$SOCK" 2>/dev/null +else + /usr/bin/lockf -t 0 /tmp/smriti-ingest.lock smriti ingest claude 2>/dev/null +fi +exit 0 +``` + +The `lockf` fallback keeps capture working if the daemon isn't running. No flag day; the old hook continues to work too. + +## Quick start + +```bash +smriti upgrade # pull the new code +smriti daemon install # register the LaunchAgent / systemd unit +smriti daemon status # PID, uptime, watched agents +``` + +That's it. Open your agent of choice, work for a while, then `smriti search` for whatever you did. The session will be there. + +## What's in the box + +- 6 daemon modules (~1500 LOC of code, ~1200 LOC of tests) +- 57 daemon tests passing (full suite: 1174+ tests) +- 9 commits on `feat/daemon-core`, all individually testable +- 1 dedicated PRD (`docs/internal/daemon-prd.md`) documenting both the design and the three pre-impl smoke-test findings that shaped it + +## Design discipline (the boring details that matter) + +Three constraints came out of pre-impl smoke tests against Bun 1.3.6, and each one shaped a design decision that would have silently bitten us in production: + +- **Single-instance enforcement uses a PID file + `kill(pid, 0)` liveness probe, not Unix-socket bind contention.** Bun's `net.createServer().listen(path)` silently succeeds on duplicate binds and steals connections from the original server. We use the PID-file pattern QMD already uses for `qmd mcp --daemon`. +- **FS watching uses native `fs.watch({ recursive: true })`, not chokidar.** Chokidar 5.0.0 under Bun fires zero events; native `fs.watch` works correctly on macOS (recursive native) and Linux (walk-and-watch). +- **DB connections are per-flush, not per-daemon-lifetime.** Repeatedly calling `ingest()` against a single long-lived SQLite handle inside one Bun process climbed to 6.8 GB peak RSS and segfaulted Bun. Opening a fresh connection per flush sidesteps this entirely; the ~30ms cost is invisible inside the 30s debounce window. + +All three findings are documented in [`docs/internal/daemon-prd.md`](https://github.com/zero8dotdev/smriti/blob/main/docs/internal/daemon-prd.md). + +## Ingest correctness — found by dogfooding + +Every non-Claude agent connector had silently drifted from the current on-disk formats. All "Sessions found: N, ingested: 0" failure modes, all fixed: + +- **Codex** — modern rollouts wrap messages in `{type:"response_item", payload:{...}}` envelopes; the parser now unwraps them and filters injected context (AGENTS.md, environment blocks). +- **Copilot (VS Code)** — chatSessions moved to `.jsonl` with `{kind:0, v:{...}}` snapshot lines, and text moved to `message.text` / response `value` fields. Discovery and parsing updated. +- **Cursor** — the big one. Real chat history lives in `globalStorage/state.vscdb` (`composerData:` + `bubbleId:` keys), not project `.cursor/` dirs. New read-only SQLite discovery with composer→workspace project mapping; `smriti ingest cursor` now works with no flags. Recovered 482 sessions / 22k messages on the dev machine. +- **Backfill correctness** — `addMessage()` honors original message timestamps (history no longer collapses to ingest day); `--force` re-ingest deletes prior messages instead of appending duplicates; `SMRITI_INGEST_NO_ENRICH=1` skips per-session LLM query expansion during bulk backfills (482 queued local-LLM inferences previously pegged the CPU for 20+ minutes). + +## Platforms + +- ✅ macOS 14+ (Apple Silicon and Intel) +- ✅ Linux (systemd-user supported) +- ⏸️ Windows — deferred to a later release. Bun's Windows daemon support is rough and named-pipe semantics differ enough from Unix sockets that we want to ship them separately rather than half-build them now. + +## Upgrading + +If you're coming from v0.6.0 / v0.7.0: + +1. `cd ~/.smriti && smriti upgrade` (or your equivalent — wherever your smriti install lives) +2. `smriti daemon install` if you want the daemon. Optional — if you skip this, Smriti continues to work exactly as it did before via the existing Claude Stop hook. + +If you do install the daemon and later decide to roll back, `smriti daemon uninstall` removes the service file and stops the daemon. The PID file and IPC socket are cleaned up automatically. There is no other state to migrate. + +## What's not in this release + +A few things explicitly deferred to keep this release tight: + +- **No real redaction pipeline.** `smriti share` still does the basic sanitization it always has. Real redaction comes in v0.8.1 / v0.9.0. +- **No read-side routing through the daemon.** `smriti search` and `smriti recall` are still one-shot CLI invocations. The daemon doesn't speed them up. +- **No auto-restart on `smriti upgrade`.** After upgrading, you'll want to `smriti daemon stop` followed by `launchctl kickstart -k gui/$UID/dev.zero8.smriti` (or `systemctl --user restart smriti` on Linux) so the daemon picks up the new code. v0.8.1 will teach `smriti upgrade` to do this automatically. + +## Thanks + +This release came out of a debugging session that found 42 stuck `smriti ingest` processes consuming 9 CPU-days. The lockf mitigation that stopped the pile-up is still in place as the fallback path; the daemon makes it usually-unnecessary. Both stories live in `docs/papers/`. + +Refs: #71, #72, #73, #74, #75. PR #76. diff --git a/docs/papers/only-by-staying.md b/docs/papers/only-by-staying.md new file mode 100644 index 0000000..9d4f235 --- /dev/null +++ b/docs/papers/only-by-staying.md @@ -0,0 +1,65 @@ +# The thing you can only know by staying + +I run `ps -ef` looking for something unrelated. The laptop fan is doing something it shouldn't. + +42 lines come back. All the same: + +``` +bun /Users/zero8/zero8.dev/smriti/src/index.ts ingest claude +``` + +The oldest one started Wednesday. It is Sunday. + +That's the moment. The moment where you realise you've been here for a while. Long enough to have written a hook script that is now eating 9 CPU-days of your laptop's life. Long enough that the script is older than your memory of writing it. Long enough that the conditions you wrote it under — three Claude sessions at most, a DB that fit in a megabyte, an ingest that returned in two seconds — are all gone, replaced by their grown-up versions you didn't notice arriving. + +## How sensible decisions accumulate + +Every line of that hook was sensible the day I wrote it. + +The hook was one line. There was no reason to add a lock — I had one Claude session at a time, and the ingest was fast, and locking adds complexity for a problem I didn't have. The "right" code that day was the simplest code that worked. + +`smriti ingest claude` was sensible too. It was a script that scanned the Claude logs directory and put new content into a SQLite DB. The directory had ten files. The DB was empty. The scan took milliseconds. There was nothing to optimise. + +And the decision to fire on every Stop event — that was the whole pitch. *Memory that's always fresh.* Asking the user to remember to ingest defeats the entire thing. Automate it. Tie it to the natural rhythm of working. + +Each of those decisions was correct in isolation, given the world at the time it was made. None of them were wrong. They just turned out to compose into something that was wrong, given the world four months later. + +## You can read about this. You can't know it. + +I have read pieces about hooks. About background work. About the difference between fire-and-forget and request/response. I have probably written some. I *knew*, in the way you know things when you've read them, that long-running operations on event triggers need backpressure. + +But I didn't feel it, the way you feel something after you've seen it eat your laptop, until I saw it eat my laptop. + +This is the part I want to write down, because I think it's the underrated thing about staying with one project for a long time. The lessons available to you change shape. At the start, the lessons are mostly external — you read someone else's blog, you copy the pattern, you avoid the trap they fell in. After a while, the lessons are mostly yours — you trip over things that were perfectly fine when you wrote them and have become broken without anyone touching them. + +## What time does + +What time does is this: it inverts which assumptions are load-bearing. + +When I started Smriti, the load-bearing assumption was "a session is one conversation in one window." The whole architecture flowed from that. As I lived with the tool, I started running three sessions, then five, then ten. Each session was sensible. Concurrency snuck in without anyone introducing it. + +When I started Smriti, the DB was small and the embedding pipeline was a plan. As I lived with the tool, both grew. The ingest that used to return in two seconds returned in two minutes. The 30-second async timeout in the hook config was a generous upper bound; then it was a tight bound; then it was meaningless. + +Nothing changed. Everything changed. + +## Living downstream + +The project doesn't tell you when its assumptions are being violated. It just gets slower and weirder, and you blame the laptop, or the day, or the model, until one day you run `ps -ef` and find your evidence. + +The people who can read this kind of evidence are not the people who studied software architecture the hardest. They're the people who stayed with one thing long enough to watch it drift away from the conditions it was written under, and to recognise the shape of that drift when it shows up somewhere else. A lot of what we call "experience" is this. Not knowing more patterns. Knowing how patterns rot. + +There's a second-order version of this, too. When I went and read QMD — the library Smriti is built on — I realised it doesn't have any of this machinery. No file watching, no debouncing, no daemon for ingest. QMD assumes the user runs `qmd update` when they want to update. The author of QMD, whoever they were the day they wrote it, decided "automatic ingest" was someone else's problem. I am now someone else. I added the automatic ingest. I now have the problem. + +That's not a critique of QMD. It's an observation about the seam where one project ends and another begins. Every "we'll just wrap this and add a little convenience" is also "we'll just inherit whatever problems this convenience creates." You can only see those problems by living downstream of the seam long enough for them to show up. The original authors couldn't have warned you. They couldn't see them either, because they hadn't stayed in your version of the world. + +## The trade + +The fix was six characters: `lockf -t 0`. The follow-up — a real daemon, FS watching, debouncing — is more involved but well-understood by now. I filed the issue. Someone (probably me) will pick it up. + +What's harder to write down is the part that happens to you while you're fixing it. The recognition that your old code is no longer your code, exactly. It belongs to a version of the project and a version of you that don't exist anymore. The new versions inherit it without remembering writing it. + +The thing that didn't exist on day one isn't the bug. The bug is just a consequence. The thing that didn't exist on day one is *the conditions under which the original code was wrong*. Those took months to arrive, quietly, while I was paying attention to other things. + +People say "this project teaches you something every day" and mean it as a compliment to the project. I think it's also a fact about time. You're not really learning *the project*. You're learning what the project becomes, slowly, without anyone making it become anything. + +The 42 processes are gone now. One command killed them all. But the version of me that wrote that hook didn't get to learn anything from them — only the version of me that found them did. That's the trade. You can't read your way to it. You have to stay. diff --git a/docs/papers/stop-hook-never-stopped.md b/docs/papers/stop-hook-never-stopped.md new file mode 100644 index 0000000..ee0c7ab --- /dev/null +++ b/docs/papers/stop-hook-never-stopped.md @@ -0,0 +1,90 @@ +# The Stop hook that never stopped + +I asked Claude Code to grep `ps -ef` for something unrelated. The output came back with 42 lines of: + +``` +bun /Users/zero8/zero8.dev/smriti/src/index.ts ingest claude +``` + +Oldest had been running since Wednesday. It was Sunday. + +Total CPU time across all 42: **13,449 minutes**. Nine CPU-days, burned silently in the background while I worked on other things. + +## What was supposed to happen + +Smriti is my cross-session memory layer for Claude Code. The mechanism is simple: + +1. Claude Code finishes a turn. +2. A `Stop` hook fires. +3. The hook runs `smriti ingest claude`, which scans `~/.claude/projects/` for new session content and writes it into a local SQLite DB. + +The hook looked like this: + +```bash +#!/bin/bash +smriti ingest claude 2>/dev/null +exit 0 +``` + +Async, 30-second timeout, fire-and-forget. Works fine if ingestion finishes inside one Claude turn. + +## What actually happened + +Ingestion does not finish inside one Claude turn — not when the DB has months of sessions, not when the embedding phase has to read every new chunk, not when SQLite contention is in play. A single run can take many minutes. + +The hook fires after **every** response, in **every** session. I had several concurrent Claude Code sessions running. Each Stop fired another ingest. None of them held a lock. So: + +- Turn ends → ingest A starts +- 30 seconds later, A is still scanning → next turn ends → ingest B starts on top of A +- B and A both grind on the same DB, contending on writes +- C joins. Then D. Then E. + +Each new process slows down the ones already running, which makes them take even longer to finish, which gives more new ones a chance to spawn before any complete. The pile-up is self-reinforcing. + +By the time I noticed, 42 processes were consuming roughly a full CPU core between them, fighting over the same SQLite file. + +## The fix + +macOS doesn't ship `flock`, but `/usr/bin/lockf` is right there and does the right thing: + +```bash +#!/bin/bash +/usr/bin/lockf -t 0 /tmp/smriti-ingest.lock smriti ingest claude 2>/dev/null +exit 0 +``` + +`-t 0` means: try to acquire the lock with a zero-second timeout. If something else already holds it, exit immediately (status 75) instead of waiting. The final `exit 0` swallows that status so Claude Code never sees a failure. + +The behavioural change: at most one ingest is ever running. If a new Stop event fires while one is in flight, the hook no-ops in 8ms. The in-flight ingest is incremental — it tracks its position in `session-resolver.ts` state — so the next un-blocked Stop picks up everything that was missed. + +Verifying: + +``` +$ /usr/bin/lockf -t 0 /tmp/lock sleep 5 & +$ /usr/bin/lockf -t 0 /tmp/lock echo "got it" +lockf: /tmp/lock: already locked +$ echo $? +75 +``` + +`lockf` on macOS uses `fcntl()` advisory locks, which the kernel releases automatically when the holding process exits — crash, kill, normal exit, doesn't matter. No stale-lock cleanup needed. + +## What I should have seen earlier + +What bothers me is that the symptom — runaway processes — is loud, but the design flaw is quiet. The hook's comment said: + +> Fires on Stop hook (after each Claude response). + +That described **the trigger**, not **the contract**. The trigger fires unconditionally, but the operation behind it isn't unconditional-safe. Any hook that kicks off work longer than the interval between fires needs one of: + +- Mutual exclusion (a lock). +- Debouncing (wait N seconds of quiet before starting). +- A queue (collapse pending fires into one). + +Defaulting to none of the above is how you end up with 9 CPU-days of duplicate work and a laptop that's been quietly running hot for four days. + +## The heuristic + +If you write a background hook that calls into a process touching shared state — a database, a network, a file — assume two will run concurrently and decide what should happen. The answer is almost never "let them both proceed." + +For one-at-a-time work, `lockf -t 0` on macOS / `flock -n` on Linux is six characters of insurance against an entire class of pile-ups. diff --git a/package.json b/package.json index 8eb379e..1e5d495 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "smriti", - "version": "0.6.0", + "version": "0.8.2", "description": "Smriti - Unified memory layer across all AI agents", "type": "module", "bin": { @@ -9,7 +9,9 @@ "scripts": { "dev": "bun --hot src/index.ts", "build": "bun build src/index.ts --outdir dist --target bun", - "test": "bun test", + "test": "bun test --cwd ./test", + "eval:recall": "bun run test/eval/recall-quality.eval.ts", + "eval:relations": "bun run test/eval/relation-inference.eval.ts", "smriti": "bun src/index.ts", "bench:qmd": "bun run scripts/bench-qmd.ts --profile ci-small --out bench/results/ci-small.json --no-llm", "bench:qmd:repeat": "bun run scripts/bench-qmd-repeat.ts --profiles ci-small,small,medium --runs 3 --out bench/results/repeat-summary.json", @@ -17,9 +19,11 @@ "bench:scorecard": "bun run scripts/bench-scorecard.ts --baseline bench/baseline.ci-small.json --profile ci-small --threshold-pct 20", "bench:ingest-hotpaths": "bun run scripts/bench-ingest-hotpaths.ts", "bench:ingest-pipeline": "bun run scripts/bench-ingest-pipeline.ts --sessions 120 --messages 12", - "release:meta": "bun run scripts/release-meta.ts" + "release:meta": "bun run scripts/release-meta.ts", + "monitor": "bun run scripts/monitor-resources.ts" }, "dependencies": { + "fast-glob": "3.3.3", "node-llama-cpp": "^3.0.0", "picomatch": "^4.0.0", "qmd": "file:./qmd" diff --git a/qmd b/qmd index d58fedf..da67604 160000 --- a/qmd +++ b/qmd @@ -1 +1 @@ -Subproject commit d58fedf4b5785ccbdcdc92f7ab7b8b175801d6e5 +Subproject commit da67604ac32f48d58177311db4f92e062d883af1 diff --git a/scripts/monitor-resources.ts b/scripts/monitor-resources.ts new file mode 100644 index 0000000..b9628a0 --- /dev/null +++ b/scripts/monitor-resources.ts @@ -0,0 +1,221 @@ +/** + * monitor-resources.ts - Sample RSS/CPU for Smriti-related processes + * (daemon, ingest, any bun src/index.ts invocation) so resource usage + * can be inspected without babysitting btop. + * + * bun run monitor watch [--pattern ] [--interval-ms 1000] [--out ] + * bun run monitor exec [--pattern ] [--interval-ms 1000] [--out ] -- + * + * `watch` samples until Ctrl+C. `exec` runs the given command to + * completion, sampling the whole time, then prints a peak-usage summary + * for every matching process seen during the run (e.g. compare the + * daemon's steady baseline against a one-off `smriti ingest claude`). + * + * Samples are always appended as CSV to --out (default .tmp/, gitignored) + * so a run can be inspected after the fact. + */ +import { appendFileSync, mkdirSync } from "fs"; +import { dirname } from "path"; + +type Sample = { + ts: number; + pid: number; + ppid: number; + rssKb: number; + cpuPct: number; + etime: string; + command: string; +}; + +type ProcSummary = { + pid: number; + command: string; + firstSeen: number; + lastSeen: number; + peakRssKb: number; + peakCpuPct: number; + sampleCount: number; +}; + +const args = process.argv.slice(2); +const flag = (name: string, fallback: string) => { + const i = args.indexOf(`--${name}`); + return i >= 0 && args[i + 1] ? args[i + 1] : fallback; +}; + +const mode = args[0] === "exec" ? "exec" : "watch"; +// Matches actual `bun ... index.ts` invocations (daemon/ingest/recall/etc), +// not just any process whose path happens to contain "smriti" (e.g. an +// editor's tsserver running out of this repo's node_modules). +const pattern = new RegExp(flag("pattern", "bun\\b.*index\\.ts"), "i"); +const intervalMs = Number(flag("interval-ms", "1000")); +const outPath = flag("out", `.tmp/resource-monitor-${Date.now()}.csv`); + +const dashIdx = args.indexOf("--"); +const execCommand = mode === "exec" ? args.slice(dashIdx + 1) : []; +if (mode === "exec" && execCommand.length === 0) { + console.error("exec mode requires a command after --, e.g.:"); + console.error(" bun run monitor exec -- smriti ingest claude"); + process.exit(1); +} + +mkdirSync(dirname(outPath), { recursive: true }); +let wroteHeader = false; + +async function sampleOnce(): Promise { + const proc = Bun.spawn(["ps", "-eo", "pid,ppid,rss,pcpu,etime,args"], { + stdout: "pipe", + }); + const text = await new Response(proc.stdout).text(); + await proc.exited; + + const ts = Date.now(); + const samples: Sample[] = []; + for (const line of text.split("\n").slice(1)) { + const m = line.match(/^\s*(\d+)\s+(\d+)\s+(\d+)\s+([\d.]+)\s+(\S+)\s+(.*)$/); + if (!m) continue; + const [, pid, ppid, rss, cpu, etime, command] = m; + if (!pattern.test(command)) continue; + if (command.includes("monitor-resources") || command.includes("ps -eo")) continue; + samples.push({ + ts, + pid: Number(pid), + ppid: Number(ppid), + rssKb: Number(rss), + cpuPct: Number(cpu), + etime, + command: command.trim(), + }); + } + return samples; +} + +function writeCsv(samples: Sample[]) { + if (samples.length === 0) return; + if (!wroteHeader) { + appendFileSync(outPath, "ts,pid,ppid,rss_kb,cpu_pct,etime,command\n"); + wroteHeader = true; + } + const rows = samples + .map((s) => `${s.ts},${s.pid},${s.ppid},${s.rssKb},${s.cpuPct},${s.etime},"${s.command.replace(/"/g, '""')}"`) + .join("\n"); + appendFileSync(outPath, rows + "\n"); +} + +function summarize(all: Sample[]): ProcSummary[] { + const byPid = new Map(); + for (const s of all) { + const existing = byPid.get(s.pid); + if (!existing) { + byPid.set(s.pid, { + pid: s.pid, + command: s.command, + firstSeen: s.ts, + lastSeen: s.ts, + peakRssKb: s.rssKb, + peakCpuPct: s.cpuPct, + sampleCount: 1, + }); + continue; + } + existing.lastSeen = s.ts; + existing.peakRssKb = Math.max(existing.peakRssKb, s.rssKb); + existing.peakCpuPct = Math.max(existing.peakCpuPct, s.cpuPct); + existing.sampleCount++; + } + return [...byPid.values()].sort((a, b) => b.peakRssKb - a.peakRssKb); +} + +function isPidAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +/** + * Wait for a spawned child to exit. Races the real `exited` promise against + * periodic liveness checks, because `child.exited` has been observed to + * never resolve even after the child process has fully terminated (no + * zombie, no remaining process) — a real hang seen during a long `smriti + * ingest --force` run. Once we've seen the pid alive at least once and then + * can no longer find it, treat that as exit rather than waiting forever. + */ +async function waitForChildExit(child: Bun.Subprocess, pollMs: number): Promise { + let sawAlive = false; + while (true) { + const raced = await Promise.race([ + child.exited.then((code) => ({ done: true as const, code })), + Bun.sleep(pollMs).then(() => ({ done: false as const, code: null })), + ]); + if (raced.done) return raced.code; + + if (isPidAlive(child.pid)) { + sawAlive = true; + continue; + } + if (sawAlive) { + console.error( + `[monitor] child.exited did not resolve after pid ${child.pid} was no longer running — treating as exited.` + ); + return null; + } + // else: pid not observed yet (spawn race at startup) — keep waiting. + } +} + +function printSummary(all: Sample[]) { + const procs = summarize(all); + if (procs.length === 0) { + console.log(`\nNo processes matched /${pattern.source}/ during this run.`); + return; + } + console.log(`\n${"PID".padEnd(8)}${"PEAK RSS".padEnd(12)}${"PEAK CPU".padEnd(10)}${"DURATION".padEnd(10)}COMMAND`); + for (const p of procs) { + const rssMb = (p.peakRssKb / 1024).toFixed(1) + " MB"; + const durationS = ((p.lastSeen - p.firstSeen) / 1000).toFixed(1) + "s"; + console.log( + `${String(p.pid).padEnd(8)}${rssMb.padEnd(12)}${(p.peakCpuPct.toFixed(1) + "%").padEnd(10)}${durationS.padEnd(10)}${p.command.slice(0, 80)}` + ); + } + console.log(`\nRaw samples: ${outPath}`); +} + +async function main() { + const allSamples: Sample[] = []; + let stopped = false; + + const loop = (async () => { + while (!stopped) { + const samples = await sampleOnce(); + allSamples.push(...samples); + writeCsv(samples); + await Bun.sleep(intervalMs); + } + })(); + + if (mode === "watch") { + console.log(`Watching /${pattern.source}/ every ${intervalMs}ms — Ctrl+C to stop and print summary.`); + process.on("SIGINT", () => { + stopped = true; + printSummary(allSamples); + process.exit(0); + }); + await loop; + } else { + console.log(`Sampling /${pattern.source}/ every ${intervalMs}ms while running: ${execCommand.join(" ")}`); + const child = Bun.spawn(execCommand, { stdout: "inherit", stderr: "inherit" }); + const exitCode = await waitForChildExit(child, Math.min(intervalMs, 500)); + stopped = true; + await loop; + printSummary(allSamples); + process.exit(exitCode ?? 0); + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/scripts/release-meta.ts b/scripts/release-meta.ts index 3da67fb..dd8d52e 100644 --- a/scripts/release-meta.ts +++ b/scripts/release-meta.ts @@ -117,7 +117,14 @@ function isConventional(c: Commit): boolean { function getCommits(rangeFrom: string | null, rangeTo: string): Commit[] { const range = rangeFrom ? `${rangeFrom}..${rangeTo}` : rangeTo; - const raw = run(`git log --no-merges --pretty=format:%H%x09%s%x09%b ${range}`); + // Not run()'s full .trim(): the oldest commit in range can have an empty + // body, leaving a trailing %x09 as the last character of the whole + // output. A full trim() strips that tab along with it, dropping that + // commit's field count below 3 and silently discarding it from + // getCommits()'s result (and thus from release notes / commit_count). + const raw = execSync(`git log --no-merges --pretty=format:%H%x09%s%x09%b ${range}`, { + encoding: "utf8", + }).replace(/\n+$/, ""); if (!raw) return []; return raw .split("\n") diff --git a/scripts/serve-report.ts b/scripts/serve-report.ts new file mode 100644 index 0000000..df95b42 --- /dev/null +++ b/scripts/serve-report.ts @@ -0,0 +1,81 @@ +/** + * serve-report.ts - Minimal local server for the learning report. + * + * bun scripts/serve-report.ts [--port 7777] [--file ] + * + * Renders the markdown client-side with marked (CDN) — no dependencies. + */ +import { existsSync } from "fs"; + +const args = process.argv.slice(2); +const flag = (name: string, fallback: string) => { + const i = args.indexOf(`--${name}`); + return i >= 0 && args[i + 1] ? args[i + 1] : fallback; +}; + +const PORT = Number(flag("port", "7777")); +const FILE = flag( + "file", + `${process.env.HOME}/zero8.dev/BUILDER-RETROSPECTIVE-2026-FEB-JUN.md` +); + +if (!existsSync(FILE)) { + console.error(`Report not found: ${FILE}`); + process.exit(1); +} + +const SHELL = ` + + + + + +Builder Retrospective — Feb–Jun 2026 + + + + + +
Loading…
+ + +`; + +Bun.serve({ + port: PORT, + hostname: "127.0.0.1", + async fetch(req) { + const { pathname } = new URL(req.url); + if (pathname === "/raw") { + return new Response(await Bun.file(FILE).text(), { + headers: { "content-type": "text/markdown; charset=utf-8" }, + }); + } + if (pathname === "/" || pathname === "/index.html") { + return new Response(SHELL, { + headers: { "content-type": "text/html; charset=utf-8" }, + }); + } + return new Response("Not found", { status: 404 }); + }, +}); + +console.log(`Serving ${FILE}`); +console.log(`→ http://127.0.0.1:${PORT}`); diff --git a/src/categorize/classifier.ts b/src/categorize/classifier.ts index 5bf17bd..347e9f5 100644 --- a/src/categorize/classifier.ts +++ b/src/categorize/classifier.ts @@ -7,7 +7,7 @@ import type { Database } from "bun:sqlite"; import { tagMessage, tagSession } from "../db"; -import { CLASSIFY_LLM_THRESHOLD, OLLAMA_HOST, OLLAMA_MODEL } from "../config"; +import { CLASSIFY_LLM_THRESHOLD, OLLAMA_HOST, requireOllamaModel } from "../config"; import { ALL_CATEGORY_IDS } from "./schema"; import { getRuleManager, type Rule } from "./rules/loader"; @@ -86,7 +86,7 @@ ${text.slice(0, 2000)}`; method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ - model: OLLAMA_MODEL, + model: requireOllamaModel(), prompt, stream: false, options: { temperature: 0.1, num_predict: 50 }, diff --git a/src/cluster.ts b/src/cluster.ts new file mode 100644 index 0000000..b17b233 --- /dev/null +++ b/src/cluster.ts @@ -0,0 +1,222 @@ +/** + * cluster.ts - Semantic session clustering (#66) + * + * k-means clustering over session embeddings (content_vectors seq=0). + * Cluster names generated by Ollama. Results persisted to smriti_session_clusters. + */ + +import type { Database } from "bun:sqlite"; +import { ollamaChat } from "./ollama"; + +// ============================================================================= +// Types +// ============================================================================= + +export type Cluster = { + id: number; + name: string; + sessionIds: string[]; + lastActive: string | null; +}; + +export type ClusterResult = { + clusters: Cluster[]; + totalSessions: number; +}; + +// ============================================================================= +// k-means Implementation +// ============================================================================= + +function cosineDistance(a: Float32Array, b: Float32Array): number { + let dot = 0, normA = 0, normB = 0; + for (let i = 0; i < a.length; i++) { + dot += a[i]! * b[i]!; + normA += a[i]! * a[i]!; + normB += b[i]! * b[i]!; + } + if (normA === 0 || normB === 0) return 1; + return 1 - dot / (Math.sqrt(normA) * Math.sqrt(normB)); +} + +function centroid(vectors: Float32Array[]): Float32Array { + if (vectors.length === 0) return new Float32Array(0); + const dim = vectors[0]!.length; + const sum = new Float32Array(dim); + for (const v of vectors) { + for (let i = 0; i < dim; i++) sum[i]! += v[i]!; + } + for (let i = 0; i < dim; i++) sum[i]! /= vectors.length; + return sum; +} + +function kmeans(vectors: Float32Array[], k: number, maxIter = 20): number[] { + if (vectors.length === 0 || k <= 0) return []; + const n = vectors.length; + k = Math.min(k, n); + + // Deterministic initialization: pick evenly spaced indices + const centroids: Float32Array[] = []; + for (let i = 0; i < k; i++) { + centroids.push(vectors[Math.floor((i * n) / k)]!); + } + + let assignments = new Array(n).fill(0); + + for (let iter = 0; iter < maxIter; iter++) { + // Assign each vector to nearest centroid + const newAssignments = vectors.map(v => { + let bestCluster = 0; + let bestDist = Infinity; + for (let c = 0; c < k; c++) { + const d = cosineDistance(v, centroids[c]!); + if (d < bestDist) { bestDist = d; bestCluster = c; } + } + return bestCluster; + }); + + // Check convergence + if (newAssignments.every((a, i) => a === assignments[i])) break; + assignments = newAssignments; + + // Update centroids + for (let c = 0; c < k; c++) { + const clusterVecs = vectors.filter((_, i) => assignments[i] === c); + if (clusterVecs.length > 0) centroids[c] = centroid(clusterVecs); + } + } + + return assignments; +} + +// ============================================================================= +// Cluster Name Generation +// ============================================================================= + +async function nameCluster(titles: string[], model?: string): Promise { + try { + const resp = await ollamaChat([ + { + role: "system", + content: + "You name topic clusters from a list of session titles. " + + "Return ONLY a concise 2-5 word phrase (no punctuation) that captures the common theme.", + }, + { + role: "user", + content: `Session titles:\n${titles.slice(0, 10).join("\n")}`, + }, + ], { temperature: 0.2, maxTokens: 16, model }); + return resp.message.content.trim().replace(/[.!?"]$/, ""); + } catch { + return `cluster-${titles[0]?.slice(0, 20) ?? "unknown"}`; + } +} + +// ============================================================================= +// Main Cluster Function +// ============================================================================= + +export async function clusterSessions( + db: Database, + options: { + projectId?: string; + k?: number; + model?: string; + } = {} +): Promise { + // Load session embeddings (first chunk per session) + const sessionQuery = options.projectId + ? `SELECT mm.session_id, cv.hash || '_' || cv.seq as hash_seq + FROM memory_messages mm + JOIN content_vectors cv ON cv.hash = mm.hash AND cv.seq = 0 + JOIN smriti_session_meta sm ON sm.session_id = mm.session_id + WHERE sm.project_id = ? + GROUP BY mm.session_id` + : `SELECT mm.session_id, cv.hash || '_' || cv.seq as hash_seq + FROM memory_messages mm + JOIN content_vectors cv ON cv.hash = mm.hash AND cv.seq = 0 + GROUP BY mm.session_id`; + + const rows = options.projectId + ? (db as any).prepare(sessionQuery).all(options.projectId) as { session_id: string; hash_seq: string }[] + : (db as any).prepare(sessionQuery).all() as { session_id: string; hash_seq: string }[]; + + if (rows.length < 2) { + return { clusters: [], totalSessions: rows.length }; + } + + // Load actual embedding vectors from vectors_vec + const sessionIds = rows.map(r => r.session_id); + const hashSeqs = rows.map(r => r.hash_seq); + + const embeddingRows = (db as any).prepare( + `SELECT hash_seq, embedding FROM vectors_vec WHERE hash_seq IN (${hashSeqs.map(() => "?").join(",")})` + ).all(...hashSeqs) as { hash_seq: string; embedding: Buffer }[]; + + const embeddingMap = new Map(embeddingRows.map(r => [r.hash_seq, new Float32Array(r.embedding.buffer)])); + + // Filter to sessions with embeddings + const validRows = rows.filter(r => embeddingMap.has(r.hash_seq)); + if (validRows.length < 2) { + return { clusters: [], totalSessions: rows.length }; + } + + const vectors = validRows.map(r => embeddingMap.get(r.hash_seq)!); + const k = options.k ?? Math.max(2, Math.min(20, Math.round(Math.sqrt(validRows.length / 2)))); + + // Run k-means + const assignments = kmeans(vectors, k); + + // Group session IDs by cluster + const clusterMap = new Map(); + for (let i = 0; i < assignments.length; i++) { + const cid = assignments[i]!; + if (!clusterMap.has(cid)) clusterMap.set(cid, []); + clusterMap.get(cid)!.push(validRows[i]!.session_id); + } + + // Load session titles + last active dates + const sessionTitleRows = (db as any).prepare( + `SELECT id, title, updated_at FROM memory_sessions WHERE id IN (${sessionIds.map(() => "?").join(",")})` + ).all(...sessionIds) as { id: string; title: string; updated_at: string }[]; + const titleMap = new Map(sessionTitleRows.map(r => [r.id, { title: r.title, updated_at: r.updated_at }])); + + // Generate cluster names + persist + (db as any).prepare(`DELETE FROM smriti_session_clusters WHERE session_id IN (${sessionIds.map(() => "?").join(",")})`).run(...sessionIds); + + const insertStmt = (db as any).prepare( + `INSERT OR REPLACE INTO smriti_session_clusters(session_id, cluster_id, cluster_name, distance) VALUES (?, ?, ?, ?)` + ); + + const clusters: Cluster[] = []; + for (const [cid, sids] of clusterMap) { + const titles = sids.map(sid => titleMap.get(sid)?.title || sid).filter(Boolean); + const name = await nameCluster(titles, options.model); + const lastActive = sids.reduce((best, sid) => { + const d = titleMap.get(sid)?.updated_at ?? null; + if (!best || (d && d > best)) return d; + return best; + }, null); + + for (const sid of sids) { + insertStmt.run(sid, cid, name, 0); + } + + clusters.push({ id: cid, name, sessionIds: sids, lastActive }); + } + + clusters.sort((a, b) => b.sessionIds.length - a.sessionIds.length); + return { clusters, totalSessions: validRows.length }; +} + +// ============================================================================= +// Cluster Lookup (for --cluster filter in recall) +// ============================================================================= + +export function getClusterSessionIds(db: Database, clusterName: string): string[] { + const rows = (db as any).prepare( + `SELECT DISTINCT session_id FROM smriti_session_clusters WHERE LOWER(cluster_name) LIKE LOWER(?)` + ).all(`%${clusterName}%`) as { session_id: string }[]; + return rows.map(r => r.session_id); +} diff --git a/src/config.ts b/src/config.ts index fde4979..2ce871b 100644 --- a/src/config.ts +++ b/src/config.ts @@ -33,12 +33,23 @@ export const CLINE_LOGS_DIR = /** GitHub Copilot (VS Code) workspaceStorage root — auto-detected per OS if not set */ export const COPILOT_STORAGE_DIR = Bun.env.COPILOT_STORAGE_DIR || ""; -/** Daemon PID file path */ +/** Cursor IDE user directory root — auto-detected per OS if not set */ +export const CURSOR_STORAGE_DIR = Bun.env.CURSOR_STORAGE_DIR || ""; + +/** Daemon PID file path. Load-bearing for single-instance enforcement. */ export const DAEMON_PID_FILE = join(HOME, ".cache", "smriti", "daemon.pid"); /** Daemon log file path */ export const DAEMON_LOG_FILE = join(HOME, ".cache", "smriti", "daemon.log"); +/** + * Daemon IPC socket path. Used for the Claude Stop hook poke. NOT used for + * single-instance enforcement — Bun's net.createServer().listen() silently + * succeeds on duplicate bind and steals connections from the original. + * Single-instance lives in DAEMON_PID_FILE + kill(pid, 0) probe instead. + */ +export const DAEMON_SOCKET_FILE = join(HOME, ".cache", "smriti", "daemon.sock"); + /** Daemon debounce interval in ms — wait this long after last file change before ingesting */ export const DAEMON_DEBOUNCE_MS = Number(Bun.env.SMRITI_DAEMON_DEBOUNCE_MS || "30000"); @@ -57,7 +68,18 @@ export const PROJECTS_ROOT = // ============================================================================= export const OLLAMA_HOST = Bun.env.OLLAMA_HOST || "http://127.0.0.1:11434"; -export const OLLAMA_MODEL = Bun.env.QMD_MEMORY_MODEL || "qwen3:8b-tuned"; +export const OLLAMA_MODEL = Bun.env.QMD_MEMORY_MODEL; + +/** Resolve the Ollama model to use, preferring an explicit override. Throws if neither is set. */ +export function requireOllamaModel(explicit?: string): string { + const model = explicit || OLLAMA_MODEL; + if (!model) { + throw new Error( + "No Ollama model configured. Set QMD_MEMORY_MODEL in your environment or .env file." + ); + } + return model; +} /** Confidence threshold below which rule-based classification triggers LLM */ export const CLASSIFY_LLM_THRESHOLD = Number( @@ -75,3 +97,7 @@ export const DEFAULT_CONTEXT_DAYS = 7; /** Git author name for team sharing */ export const AUTHOR = Bun.env.SMRITI_AUTHOR || Bun.env.USER || "unknown"; + +/** Directory for session markdown documents (QMD smriti-sessions collection) */ +export const SMRITI_SESSIONS_DIR = + Bun.env.SMRITI_SESSIONS_DIR || join(HOME, ".cache", "smriti", "sessions"); diff --git a/src/daemon/client.ts b/src/daemon/client.ts new file mode 100644 index 0000000..fa8a021 --- /dev/null +++ b/src/daemon/client.ts @@ -0,0 +1,89 @@ +/** + * daemon/client.ts - Socket-less client primitives for `smriti daemon stop` + * and `smriti daemon status`. + * + * Both commands work entirely through the PID file. They don't need to + * connect to the IPC socket. This keeps the lifecycle commands working + * even if the socket file is stale or the daemon is wedged in a way + * that makes it unresponsive on the socket. + * + * - stop: read the PID, send SIGTERM, poll for exit. + * - status: read the PID, probe liveness via kill(pid, 0), surface + * the PID-file mtime as a coarse "running since" indicator. + */ + +import { statSync } from "node:fs"; + +import { DAEMON_PID_FILE } from "../config"; +import { detectRunningDaemon } from "./server"; + +export type DaemonStatus = { + /** True if a live daemon process owns the PID file. */ + running: boolean; + /** Daemon PID, or null if not running. */ + pid: number | null; + /** + * When the PID file was written, as a Date. Used as a proxy for + * "when the daemon started." Null if no PID file exists. + */ + startedAt: Date | null; + /** Path of the PID file inspected. Useful for error messages. */ + pidFile: string; +}; + +export function getDaemonStatus(): DaemonStatus { + const pid = detectRunningDaemon(); + if (pid === null) { + return { running: false, pid: null, startedAt: null, pidFile: DAEMON_PID_FILE }; + } + let startedAt: Date | null = null; + try { + const stat = statSync(DAEMON_PID_FILE); + // birthtime is set on macOS and most modern Linux filesystems. Fall + // back to ctime if birthtime is the epoch. + startedAt = stat.birthtime.getTime() === 0 ? stat.ctime : stat.birthtime; + } catch { + // PID file vanished between the detect call and statSync; the daemon + // is racing us out of existence. Report as not-running for safety. + return { running: false, pid: null, startedAt: null, pidFile: DAEMON_PID_FILE }; + } + return { running: true, pid, startedAt, pidFile: DAEMON_PID_FILE }; +} + +export type StopResult = + | { state: "stopped"; pid: number } + | { state: "not-running" } + | { state: "timeout"; pid: number }; + +/** + * Stop the running daemon by sending SIGTERM and waiting for the PID + * file to disappear (which the daemon's signal handler does as part of + * graceful shutdown). + * + * Returns: + * - { state: "stopped", pid } if the daemon exited within the timeout. + * - { state: "not-running" } if no daemon was running to begin with. + * - { state: "timeout", pid } if the daemon didn't exit in time. + * Caller may want to escalate (SIGKILL) or surface to the user. + */ +export async function stopDaemon(opts: { timeoutMs?: number; pollMs?: number } = {}): Promise { + const pid = detectRunningDaemon(); + if (pid === null) return { state: "not-running" }; + + try { + process.kill(pid, "SIGTERM"); + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === "ESRCH") return { state: "not-running" }; + throw err; + } + + const timeoutMs = opts.timeoutMs ?? 5000; + const pollMs = opts.pollMs ?? 100; + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (detectRunningDaemon() === null) return { state: "stopped", pid }; + await new Promise((r) => setTimeout(r, pollMs)); + } + return { state: "timeout", pid }; +} diff --git a/src/daemon/handlers.ts b/src/daemon/handlers.ts new file mode 100644 index 0000000..672f8e1 --- /dev/null +++ b/src/daemon/handlers.ts @@ -0,0 +1,67 @@ +/** + * daemon/handlers.ts - FS-event → agent routing helpers. + * + * Pure helpers that turn a file path into the agent name responsible for + * that path, and produce the list of (agent, root) pairs the daemon + * should watch. + * + * For v0.8.0 the routing is coarse: we map by agent, not by project. + * The downside is that a change in one Claude project causes a scan + * across all Claude projects; the upside is that ingest() is already + * incremental, so unchanged sessions cost almost nothing. A per-project + * resolution layer can replace this without changing the daemon shape. + */ + +import { existsSync } from "node:fs"; +import { sep } from "node:path"; + +import { + CLAUDE_LOGS_DIR, + CODEX_LOGS_DIR, + CLINE_LOGS_DIR, + COPILOT_STORAGE_DIR, +} from "../config"; + +export type AgentRoot = { + /** Stable agent identifier matching the value passed to `ingest(db, agent)`. */ + agent: string; + /** Absolute filesystem root the daemon will watch for this agent. */ + root: string; +}; + +/** + * Return the default agent roots the daemon should watch. + * Filters out roots that don't currently exist on disk so we don't + * crash trying to watch a Codex install that isn't there. + */ +export function getDefaultAgentRoots(): AgentRoot[] { + const candidates: AgentRoot[] = [ + { agent: "claude", root: CLAUDE_LOGS_DIR }, + { agent: "codex", root: CODEX_LOGS_DIR }, + { agent: "cline", root: CLINE_LOGS_DIR }, + ]; + if (COPILOT_STORAGE_DIR) { + candidates.push({ agent: "copilot", root: COPILOT_STORAGE_DIR }); + } + return candidates.filter((c) => c.root && existsSync(c.root)); +} + +/** + * Given an absolute file path and a list of agent roots, return the agent + * that owns the path, or null if no root contains it. + * + * Matching is done by string prefix with a trailing separator guard to + * avoid false positives like `~/.claude/projects-archive/` matching + * `~/.claude/projects/`. + */ +export function resolveAgentForPath( + path: string, + roots: AgentRoot[], +): string | null { + for (const { agent, root } of roots) { + if (!root) continue; + if (path === root) return agent; + if (path.startsWith(root + sep)) return agent; + } + return null; +} diff --git a/src/daemon/index.ts b/src/daemon/index.ts new file mode 100644 index 0000000..52b809a --- /dev/null +++ b/src/daemon/index.ts @@ -0,0 +1,201 @@ +/** + * daemon/index.ts - Top-level daemon entry point. + * + * Wires the five core modules into one process: + * + * FS event ─► resolveAgentForPath ─► queue.schedule ─┐ + * ├─► onFlush ─► fresh DB ─► ingest() ─► close DB + * hook poke ─► server.onPoke ─► queue.flush("claude")─┘ + * + * Open a fresh SQLite handle per flush (not once at boot) per smoke-test + * finding 3 — repeatedly calling ingest() against a single long-lived + * connection segfaulted Bun 1.3.6. See docs/internal/daemon-prd.md. + * + * The runDaemon function is dependency-injectable: tests pass a mock + * flushAgent so they can verify the wiring without touching the user's + * real DB or spawning real agent log files. + */ + +import type { Database } from "bun:sqlite"; +import { QMD_DB_PATH } from "../config"; +import { initSmriti, closeDb } from "../db"; +import { ingest } from "../ingest"; +import { startDaemon as startServer, type DaemonHandle } from "./server"; +import { watchRecursive, type WatcherHandle } from "./watcher"; +import { createDebounceQueue, type DebounceQueueHandle } from "./queue"; +import { + getDefaultAgentRoots, + resolveAgentForPath, + type AgentRoot, +} from "./handlers"; + +export type RunDaemonOptions = { + /** Override the agent roots to watch. Defaults to getDefaultAgentRoots(). */ + agentRoots?: AgentRoot[]; + /** + * Override the per-flush callback. Defaults to opening a fresh DB + * handle, calling ingest(db, agent), and closing the handle. Tests + * use this to verify wiring without invoking the real ingest path. + */ + flushAgent?: (agent: string) => void | Promise; + /** Optional logger. Defaults to console.error (so stderr, not stdout). */ + log?: (msg: string) => void; + /** Override the per-project debounce window in ms. */ + debounceMs?: number; + /** + * Let routine daemon ingest trigger LLM query-expansion enrichment. + * Defaults to false: the daemon is parse-and-write only (per + * docs/internal/daemon-prd.md #93 — no embedding model in the daemon). + * Enrichment is a manual `smriti enrich` concern; leaving this on + * caused the recurring "LlamaGrammar ... different Llama instance" + * crash when combined with a fresh LlamaCpp per flush. + */ + enrichOnIngest?: boolean; +}; + +export type RunningDaemon = { + /** Daemon PID. */ + pid: number; + /** Number of agent roots being watched. */ + watchedAgents: string[]; + /** Stop the daemon and release all resources. Idempotent. */ + shutdown(): Promise; +}; + +/** + * Chain every flush onto this promise so at most one is ever in flight. + * The debounce timers in queue.ts stay independent per project — this only + * serializes the actual open-DB/ingest/close body. Required because: + * 1. initSmriti()/closeDb() go through module-level singletons in + * src/db.ts and src/store.ts shared across the whole process — two + * concurrent flushes would stomp on each other's connection. + * 2. Disposing one flush's LlamaCpp/Llama backend (see closeDb below) + * while another flush's is still loading/active can abort the + * process with a native GGML_ASSERT in ggml-metal — reproduced when + * multiple LlamaCpp instances were constructed/disposed concurrently + * in one process during verification of this fix. + */ +let flushChain: Promise = Promise.resolve(); + +/** + * Default per-flush behavior: open SQLite, call ingest(), close SQLite. + * Errors are logged but not rethrown — one bad flush should not crash + * the daemon. + */ +async function defaultFlushAgent(agent: string, log: (m: string) => void): Promise { + const run = async (): Promise => { + let db: Database; + try { + db = await initSmriti(QMD_DB_PATH); + } catch (err) { + log(`[flush ${agent}] failed to open DB: ${(err as Error).message}`); + return; + } + try { + const r = await ingest(db, agent); + log( + `[flush ${agent}] ingested=${r.sessionsIngested}/${r.sessionsFound}, ` + + `msgs=${r.messagesIngested}, errs=${r.errors.length}`, + ); + } catch (err) { + log(`[flush ${agent}] ingest failed: ${(err as Error).message}`); + } finally { + // closeDb() (not a raw db.close()) so the QMD store's LlamaCpp/Llama + // backend is disposed too, not just the SQLite handle — otherwise + // each flush's LlamaCpp instance is orphaned to a 5-min inactivity + // timer instead of being freed immediately. + try { await closeDb(); } catch {} + } + }; + + const next = flushChain.then(run, run); + flushChain = next; + return next; +} + +/** + * Start the daemon. Returns once everything is wired and the IPC + * socket is bound. Throws if another daemon is already running. + * + * Caller is responsible for keeping the process alive. Typical usage + * is to call this from `smriti daemon` and never return — the daemon + * process lives until SIGTERM / SIGINT, at which point the installed + * signal handlers from server.ts trigger a graceful shutdown. + */ +export async function runDaemon(opts: RunDaemonOptions = {}): Promise { + const log = opts.log ?? ((msg: string) => console.error(`[smriti] ${msg}`)); + + // The daemon does parse-and-write only — LLM enrichment (query expansion) + // is a manual `smriti enrich` concern, not something ingest should trigger + // as a side effect of every flush. See docs/internal/daemon-prd.md #93 and + // the recurring "LlamaGrammar ... different Llama instance" crash this + // caused when combined with a fresh LlamaCpp per flush. + if (opts.enrichOnIngest !== true) { + process.env.SMRITI_INGEST_NO_ENRICH = "1"; + } + + const agentRoots = opts.agentRoots ?? getDefaultAgentRoots(); + + if (agentRoots.length === 0) { + log( + "no agent roots found — none of ~/.claude/projects, ~/.codex, ~/.cline/tasks " + + "exist on this machine. Daemon will still run but won't capture anything.", + ); + } + + const flushAgent = opts.flushAgent ?? ((agent: string) => defaultFlushAgent(agent, log)); + + const queue: DebounceQueueHandle = createDebounceQueue({ + debounceMs: opts.debounceMs, + onFlush: flushAgent, + log, + }); + + let server: DaemonHandle; + try { + server = await startServer({ + log, + onPoke: () => { + // The Claude Stop hook is the only thing that pokes today, so we + // route every poke to a Claude flush. If we ever grow other-agent + // hooks, the protocol gets a one-byte agent id. + void queue.flush("claude"); + }, + }); + } catch (err) { + queue.close(); + throw err; + } + + const watchers: WatcherHandle[] = []; + for (const { agent, root } of agentRoots) { + try { + const w = watchRecursive(root, (event) => { + const resolved = resolveAgentForPath(event.path, agentRoots); + if (resolved === null) return; // event not under any watched root (shouldn't happen) + queue.schedule(resolved); + }); + watchers.push(w); + log(`watching ${agent} at ${root}`); + } catch (err) { + log(`failed to watch ${agent} at ${root}: ${(err as Error).message}`); + } + } + + let shutdownPromise: Promise | null = null; + return { + pid: server.pid, + watchedAgents: agentRoots.map((r) => r.agent), + shutdown(): Promise { + if (shutdownPromise) return shutdownPromise; + shutdownPromise = (async () => { + for (const w of watchers) { + try { w.close(); } catch {} + } + queue.close(); + await server.shutdown(); + })(); + return shutdownPromise; + }, + }; +} diff --git a/src/daemon/install.ts b/src/daemon/install.ts new file mode 100644 index 0000000..b1dd272 --- /dev/null +++ b/src/daemon/install.ts @@ -0,0 +1,331 @@ +/** + * daemon/install.ts - Service-file generation and lifecycle install/uninstall. + * + * On macOS, generates a LaunchAgent plist at + * ~/Library/LaunchAgents/dev.zero8.smriti.plist + * and registers it via launchctl, so the daemon starts at user login + * and restarts on crash. + * + * On Linux, generates a systemd user unit at + * ~/.config/systemd/user/smriti.service + * and registers it via systemctl --user. Same semantics: starts at + * login, restarts on crash. + * + * Idempotent: re-installing produces the same files and registered + * service. Uninstalling is the inverse — it tears down the registration + * and removes the unit file, leaving the system in its pre-install state. + * + * The pure template generators (generatePlist, generateSystemdUnit) are + * exported so they can be unit-tested without invoking launchctl/systemctl. + * The runner abstraction (RunCmd) lets the integration paths be tested + * with a mock command runner. + */ + +import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { dirname, join } from "node:path"; + +import { DAEMON_LOG_FILE } from "../config"; + +export const SMRITI_LABEL = "dev.zero8.smriti"; + +/** The path used to launch the daemon — typically Bun + the script path. */ +export type LaunchTarget = { + /** Executable to invoke (usually process.execPath / the bun binary). */ + exec: string; + /** Arguments after the executable, ending with "daemon". */ + args: string[]; +}; + +export type InstallTarget = { + platform: "darwin" | "linux"; + /** Path to the service file the installer will write. */ + servicePath: string; +}; + +// Pure template generators -------------------------------------------------- + +export function generatePlist(opts: { launch: LaunchTarget; logFile: string }): string { + const args = [opts.launch.exec, ...opts.launch.args]; + const programArguments = args + .map((a) => ` ${escapeXml(a)}`) + .join("\n"); + return ` + + + + Label + ${SMRITI_LABEL} + ProgramArguments + +${programArguments} + + RunAtLoad + + KeepAlive + + StandardOutPath + ${escapeXml(opts.logFile)} + StandardErrorPath + ${escapeXml(opts.logFile)} + ProcessType + Background + + +`; +} + +export function generateSystemdUnit(opts: { launch: LaunchTarget }): string { + const execStart = [opts.launch.exec, ...opts.launch.args] + .map(quoteShellArg) + .join(" "); + return `[Unit] +Description=Smriti daemon — cross-agent capture +After=default.target + +[Service] +Type=simple +ExecStart=${execStart} +Restart=on-failure +RestartSec=5 +# Keep the daemon nice — it's background indexing, not anything urgent. +Nice=10 +IOSchedulingClass=idle + +[Install] +WantedBy=default.target +`; +} + +// Platform / target resolution --------------------------------------------- + +export function resolveInstallTarget(): InstallTarget { + const platform = process.platform; + if (platform === "darwin") { + return { + platform: "darwin", + servicePath: join(homedir(), "Library", "LaunchAgents", `${SMRITI_LABEL}.plist`), + }; + } + if (platform === "linux") { + return { + platform: "linux", + servicePath: join(homedir(), ".config", "systemd", "user", "smriti.service"), + }; + } + throw new Error( + `Unsupported platform ${platform}. Smriti daemon currently supports macOS and Linux only.`, + ); +} + +/** + * Resolve the executable + args that the service file will invoke. + * Defaults to process.execPath (bun) + the current process.argv[1] + * (the smriti entry point, whatever the user invoked us with) + + * the "daemon" subcommand. + */ +export function resolveLaunchTarget(): LaunchTarget { + const script = process.argv[1]; + if (!script) { + throw new Error("Could not resolve script path from process.argv[1] — refusing to write a broken service file."); + } + return { + exec: process.execPath, + args: [script, "daemon"], + }; +} + +// Runner abstraction ------------------------------------------------------- + +export type RunResult = { code: number; stdout: string; stderr: string }; +export type RunCmd = (cmd: string, args: string[]) => Promise; + +export const defaultRunCmd: RunCmd = async (cmd, args) => { + const proc = Bun.spawn([cmd, ...args], { stdout: "pipe", stderr: "pipe" }); + await proc.exited; + const stdout = await new Response(proc.stdout).text(); + const stderr = await new Response(proc.stderr).text(); + return { code: proc.exitCode ?? 0, stdout, stderr }; +}; + +// Install / uninstall ------------------------------------------------------ + +export type InstallOptions = { + /** Override the launch target. Useful for tests + ad-hoc deployments. */ + launch?: LaunchTarget; + /** Override the install target. Useful for tests. */ + target?: InstallTarget; + /** Override the log file path. */ + logFile?: string; + /** Override the command runner. Useful for tests. */ + run?: RunCmd; + /** Logger. */ + log?: (msg: string) => void; + /** If true, overwrite an existing service file even if it's already there. */ + force?: boolean; +}; + +export type InstallResult = { + servicePath: string; + wrote: boolean; + alreadyRegistered: boolean; +}; + +export async function installDaemon(opts: InstallOptions = {}): Promise { + const target = opts.target ?? resolveInstallTarget(); + const launch = opts.launch ?? resolveLaunchTarget(); + const logFile = opts.logFile ?? DAEMON_LOG_FILE; + const run = opts.run ?? defaultRunCmd; + const log = opts.log ?? ((m: string) => console.error(`[install] ${m}`)); + + // Ensure parent dir exists. + mkdirSync(dirname(target.servicePath), { recursive: true }); + mkdirSync(dirname(logFile), { recursive: true }); + + const content = + target.platform === "darwin" + ? generatePlist({ launch, logFile }) + : generateSystemdUnit({ launch }); + + let wrote = false; + if (!existsSync(target.servicePath) || opts.force) { + writeFileSync(target.servicePath, content); + wrote = true; + log(`wrote ${target.servicePath}`); + } else { + // If the contents already match what we'd write, skip the rewrite + // and the registration toggle. This is the "idempotent" path. + const existing = readFileSync(target.servicePath, "utf-8"); + if (existing === content) { + log(`service file unchanged at ${target.servicePath}`); + } else { + log( + `service file at ${target.servicePath} differs from generated content; ` + + `pass --force to overwrite. Skipping.`, + ); + return { servicePath: target.servicePath, wrote: false, alreadyRegistered: false }; + } + } + + // Register / reload. + if (target.platform === "darwin") { + // Try the modern launchctl invocation first; fall back to legacy if needed. + const uid = process.getuid?.() ?? 0; + const domain = `gui/${uid}`; + const bootstrap = await run("launchctl", ["bootstrap", domain, target.servicePath]); + if (bootstrap.code === 0) { + log(`launchctl bootstrap ok`); + return { servicePath: target.servicePath, wrote, alreadyRegistered: false }; + } + // Bootstrap fails with code 17 (EEXIST) if the label is already loaded. + // That's the "already registered" path — not an error. + if (bootstrap.stderr.includes("already") || bootstrap.code === 17) { + log(`launchctl bootstrap: already registered`); + return { servicePath: target.servicePath, wrote, alreadyRegistered: true }; + } + // Older macOS: fall back to launchctl load. + const load = await run("launchctl", ["load", "-w", target.servicePath]); + if (load.code === 0) { + log(`launchctl load ok`); + return { servicePath: target.servicePath, wrote, alreadyRegistered: false }; + } + throw new Error( + `launchctl registration failed: bootstrap exit=${bootstrap.code} stderr=${bootstrap.stderr.trim()}, ` + + `load exit=${load.code} stderr=${load.stderr.trim()}`, + ); + } + + // Linux + const reload = await run("systemctl", ["--user", "daemon-reload"]); + if (reload.code !== 0) { + throw new Error(`systemctl daemon-reload failed: ${reload.stderr.trim()}`); + } + const enable = await run("systemctl", ["--user", "enable", "--now", "smriti"]); + if (enable.code !== 0) { + throw new Error(`systemctl enable --now smriti failed: ${enable.stderr.trim()}`); + } + log(`systemctl enable --now smriti ok`); + return { servicePath: target.servicePath, wrote, alreadyRegistered: false }; +} + +export type UninstallOptions = { + target?: InstallTarget; + run?: RunCmd; + log?: (msg: string) => void; +}; + +export type UninstallResult = { + servicePath: string; + removedFile: boolean; + unregistered: boolean; +}; + +export async function uninstallDaemon(opts: UninstallOptions = {}): Promise { + const target = opts.target ?? resolveInstallTarget(); + const run = opts.run ?? defaultRunCmd; + const log = opts.log ?? ((m: string) => console.error(`[uninstall] ${m}`)); + + let unregistered = false; + if (target.platform === "darwin") { + const uid = process.getuid?.() ?? 0; + const bootout = await run("launchctl", ["bootout", `gui/${uid}/${SMRITI_LABEL}`]); + if (bootout.code === 0) { + unregistered = true; + log(`launchctl bootout ok`); + } else if (existsSync(target.servicePath)) { + // Fall back to unload. + const unload = await run("launchctl", ["unload", target.servicePath]); + if (unload.code === 0) { + unregistered = true; + log(`launchctl unload ok`); + } else { + log(`launchctl unregister failed (continuing): bootout=${bootout.code} unload=${unload.code}`); + } + } else { + log(`launchctl bootout exit=${bootout.code} (no service file to fall back to)`); + } + } else if (target.platform === "linux") { + const disable = await run("systemctl", ["--user", "disable", "--now", "smriti"]); + if (disable.code === 0) { + unregistered = true; + log(`systemctl disable --now ok`); + } else { + log(`systemctl disable --now failed (continuing): ${disable.stderr.trim()}`); + } + } + + let removedFile = false; + if (existsSync(target.servicePath)) { + try { + unlinkSync(target.servicePath); + removedFile = true; + log(`removed ${target.servicePath}`); + } catch (err) { + log(`failed to remove ${target.servicePath}: ${(err as Error).message}`); + } + } + + // On Linux, daemon-reload to forget the unit. + if (target.platform === "linux") { + await run("systemctl", ["--user", "daemon-reload"]); + } + + return { servicePath: target.servicePath, removedFile, unregistered }; +} + +// Helpers ------------------------------------------------------------------- + +function escapeXml(s: string): string { + return s + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); +} + +function quoteShellArg(s: string): string { + // systemd ExecStart accepts double-quoted strings with backslash-escaping. + if (/^[A-Za-z0-9_\-./]+$/.test(s)) return s; + return `"${s.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`; +} diff --git a/src/daemon/queue.ts b/src/daemon/queue.ts new file mode 100644 index 0000000..31e93b1 --- /dev/null +++ b/src/daemon/queue.ts @@ -0,0 +1,101 @@ +/** + * daemon/queue.ts - Per-project debounce queue. + * + * The daemon receives a stream of filesystem events from the watcher and + * a stream of pokes from the Stop hook. Both feed into this queue, which + * coalesces bursts and triggers a single ingest per project per quiet + * window. + * + * Why per-project, not global: a busy session in project A shouldn't + * delay project B's ingest. Each project gets its own timer; resetting + * one doesn't affect the others. + * + * The queue is intentionally decoupled from the ingest call itself. + * Callers wire `onFlush` to whatever should happen when a project goes + * quiet — typically: open a fresh SQLite handle, call ingest(), + * close the handle. (Per smoke-test finding 3, we do not reuse a + * single DB connection across flushes.) + */ + +import { DAEMON_DEBOUNCE_MS } from "../config"; + +export type DebounceQueueOptions = { + /** Debounce window in ms. Defaults to DAEMON_DEBOUNCE_MS (30s). */ + debounceMs?: number; + /** Called when a project's debounce timer fires. May be async. */ + onFlush: (projectId: string) => void | Promise; + /** Optional logger. Defaults to a no-op. */ + log?: (msg: string) => void; +}; + +export type DebounceQueueHandle = { + /** + * Reset the debounce timer for this project. If a timer is already + * running, it's cleared and restarted. Otherwise a new timer is + * scheduled. + */ + schedule(projectId: string): void; + /** + * Immediately invoke onFlush for this project, canceling any pending + * debounce. Returns once onFlush resolves (or throws — errors are + * propagated, the caller decides how to handle them). + */ + flush(projectId: string): Promise; + /** Number of projects with a pending debounce timer. */ + pending(): number; + /** Whether this project has a pending timer. */ + isPending(projectId: string): boolean; + /** Cancel all pending timers. Pending onFlush callbacks are NOT executed. */ + close(): void; +}; + +export function createDebounceQueue(opts: DebounceQueueOptions): DebounceQueueHandle { + const debounceMs = opts.debounceMs ?? DAEMON_DEBOUNCE_MS; + const log = opts.log ?? (() => {}); + const timers = new Map>(); + let closed = false; + + const runFlush = async (projectId: string) => { + timers.delete(projectId); + try { + await opts.onFlush(projectId); + } catch (err) { + log(`onFlush error for ${projectId}: ${(err as Error).message}`); + } + }; + + return { + schedule(projectId: string) { + if (closed) return; + const existing = timers.get(projectId); + if (existing) clearTimeout(existing); + const handle = setTimeout(() => { void runFlush(projectId); }, debounceMs); + // Don't keep Node alive solely for this timer — daemon process lifetime + // is owned by the server, not the queue. + if (typeof handle.unref === "function") handle.unref(); + timers.set(projectId, handle); + }, + + async flush(projectId: string) { + if (closed) return; + const existing = timers.get(projectId); + if (existing) clearTimeout(existing); + timers.delete(projectId); + await opts.onFlush(projectId); + }, + + pending() { + return timers.size; + }, + + isPending(projectId: string) { + return timers.has(projectId); + }, + + close() { + closed = true; + for (const t of timers.values()) clearTimeout(t); + timers.clear(); + }, + }; +} diff --git a/src/daemon/server.ts b/src/daemon/server.ts new file mode 100644 index 0000000..e2eaabd --- /dev/null +++ b/src/daemon/server.ts @@ -0,0 +1,172 @@ +/** + * daemon/server.ts - Smriti daemon server core. + * + * Owns: + * - Single-instance enforcement via PID file + kill(pid, 0) liveness probe. + * We use this pattern instead of Unix-socket bind contention because Bun's + * net.createServer().listen(path) silently succeeds on duplicate binds and + * steals connections from the original — verified during pre-impl smoke + * tests. See docs/internal/daemon-prd.md. + * - The IPC Unix socket for hook pokes (one server, many short-lived clients). + * - SIGTERM / SIGINT graceful shutdown with PID-file and socket-file cleanup. + * + * Out of scope (handled in other modules): + * - FS watching (watcher.ts) + * - Per-project debounce queue (queue.ts) + * - Ingest dispatch + * - Stop / status CLI client (client.ts) + */ + +import { existsSync, readFileSync, writeFileSync, unlinkSync, mkdirSync } from "node:fs"; +import { dirname } from "node:path"; +import { createServer, type Server } from "node:net"; + +import { + DAEMON_PID_FILE, + DAEMON_SOCKET_FILE, +} from "../config"; + +export type DaemonHandle = { + /** PID of the running daemon (this process). */ + pid: number; + /** Path of the bound IPC socket. */ + socketPath: string; + /** Path of the PID file. */ + pidFile: string; + /** Graceful shutdown — close socket, remove PID + socket files. Idempotent. */ + shutdown(): Promise; +}; + +export type DaemonOptions = { + /** Called when a poke is received on the IPC socket. */ + onPoke?: () => void | Promise; + /** Override the default console logger. */ + log?: (msg: string) => void; +}; + +/** + * Return the PID of the currently running daemon, or null if none is running. + * + * Reads DAEMON_PID_FILE. If it exists and the named process is alive + * (probed via kill(pid, 0)), returns that PID. If the PID file exists but + * the process is gone (ESRCH), the stale PID file is unlinked and null is + * returned. Garbage PID files are likewise cleaned and treated as absent. + */ +export function detectRunningDaemon(): number | null { + if (!existsSync(DAEMON_PID_FILE)) return null; + + let raw: string; + try { + raw = readFileSync(DAEMON_PID_FILE, "utf-8").trim(); + } catch { + return null; + } + + const pid = Number.parseInt(raw, 10); + if (!Number.isFinite(pid) || pid <= 0) { + // Garbage PID file — clean it up so the next start succeeds. + try { unlinkSync(DAEMON_PID_FILE); } catch {} + return null; + } + + try { + process.kill(pid, 0); // signal 0 is a liveness probe; throws if no such process + return pid; + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === "ESRCH") { + // Process is gone. Stale PID file — clean and report not-running. + try { unlinkSync(DAEMON_PID_FILE); } catch {} + return null; + } + // EPERM means the process exists but is owned by someone else. Treat as + // running — we should not start a second daemon on top of it. + if (code === "EPERM") return pid; + throw err; + } +} + +/** + * Start the daemon in the current process. Returns a handle whose shutdown() + * cleans up the PID file and IPC socket. Throws if another daemon is already + * running (its PID is included in the error message). + * + * Caller is responsible for keeping the process alive — startDaemon itself + * returns once the socket is bound. Typical usage is to call this from a + * long-running entry point (`smriti daemon`) and never return. + */ +export async function startDaemon(opts: DaemonOptions = {}): Promise { + const existing = detectRunningDaemon(); + if (existing !== null) { + throw new Error(`Smriti daemon already running (PID ${existing})`); + } + + const pid = process.pid; + const log = opts.log ?? ((msg: string) => console.log(`[daemon ${pid}] ${msg}`)); + + // Ensure the cache directory exists (creates both pid and socket parents). + mkdirSync(dirname(DAEMON_PID_FILE), { recursive: true }); + + // Write our PID. From this point on, anyone calling detectRunningDaemon() + // will see us as the running daemon. + writeFileSync(DAEMON_PID_FILE, `${pid}\n`); + log(`started, pid=${pid}`); + + // Clean any stale socket file from a previous crash. We've already + // established (via PID-file check above) that no live daemon owns it. + try { unlinkSync(DAEMON_SOCKET_FILE); } catch {} + + const onPoke = opts.onPoke ?? (() => log("got poke")); + + const server: Server = createServer((conn) => { + // Drain any incoming bytes (the poke protocol is "any connection wakes us"). + conn.on("data", () => {}); + conn.on("error", () => {}); + conn.on("end", () => {}); + Promise.resolve(onPoke()).catch((e: Error) => log(`poke handler error: ${e.message}`)); + conn.end(); + }); + + await new Promise((resolve, reject) => { + const onErr = (err: Error) => { server.removeListener("listening", onOk); reject(err); }; + const onOk = () => { server.removeListener("error", onErr); resolve(); }; + server.once("error", onErr); + server.once("listening", onOk); + server.listen(DAEMON_SOCKET_FILE); + }); + + log(`bound socket at ${DAEMON_SOCKET_FILE}`); + + let shutdownPromise: Promise | null = null; + const handle: DaemonHandle = { + pid, + socketPath: DAEMON_SOCKET_FILE, + pidFile: DAEMON_PID_FILE, + shutdown(): Promise { + if (shutdownPromise) return shutdownPromise; + shutdownPromise = (async () => { + log("shutting down"); + await new Promise((resolve) => server.close(() => resolve())); + try { unlinkSync(DAEMON_SOCKET_FILE); } catch {} + try { unlinkSync(DAEMON_PID_FILE); } catch {} + log("stopped"); + })(); + return shutdownPromise; + }, + }; + + // Install signal handlers for graceful shutdown. Exit with conventional + // signal-derived status (128 + signo) so process supervisors can tell. + const installSignal = (sig: "SIGINT" | "SIGTERM", signo: number) => { + process.on(sig, () => { + log(`received ${sig}`); + handle.shutdown() + .then(() => process.exit(128 + signo)) + .catch((e: Error) => { console.error("shutdown error:", e); process.exit(1); }); + }); + }; + installSignal("SIGINT", 2); + installSignal("SIGTERM", 15); + + return handle; +} diff --git a/src/daemon/watcher.ts b/src/daemon/watcher.ts new file mode 100644 index 0000000..0a6a47d --- /dev/null +++ b/src/daemon/watcher.ts @@ -0,0 +1,124 @@ +/** + * daemon/watcher.ts - Recursive directory watcher. + * + * Wraps Node's fs.watch to fire a single typed event per filesystem change + * across a directory tree. macOS gets recursive watching for free; Linux's + * inotify backend doesn't implement `recursive`, so we walk the tree at + * startup and watch each directory, re-watching on dir-create events. + * + * We deliberately use native fs.watch instead of chokidar because chokidar + * 5.0.0 fired zero events under Bun 1.3.6 during pre-impl smoke testing. + * Native fs.watch correctly fires for both new file creation (`rename`) + * and content changes (`change`). See docs/internal/daemon-prd.md. + */ + +import { watch, type FSWatcher, readdirSync, statSync, existsSync } from "node:fs"; +import { join, resolve } from "node:path"; + +export type WatcherEvent = { + /** + * fs.watch event type. `rename` covers file create/delete/move; `change` + * covers in-place content modification. + */ + type: "rename" | "change"; + /** Absolute path of the changed entry. */ + path: string; +}; + +export type WatcherHandle = { + /** Number of directories currently being watched (relevant on Linux). */ + watchedCount(): number; + /** Stop watching and release all FSWatcher instances. */ + close(): void; +}; + +const IS_MACOS = process.platform === "darwin"; +const IS_WINDOWS = process.platform === "win32"; + +/** + * Watch a directory tree recursively. The callback fires for every fs.watch + * event under `root`. Paths in events are absolute. + * + * The root must exist when watchRecursive is called; this is not a "watch + * for the root to appear" primitive. + */ +export function watchRecursive( + root: string, + onEvent: (event: WatcherEvent) => void, +): WatcherHandle { + const absRoot = resolve(root); + if (!existsSync(absRoot)) { + throw new Error(`watchRecursive: root does not exist: ${absRoot}`); + } + + // Native recursive support on macOS and Windows. On Linux we walk + watch. + const useNativeRecursive = IS_MACOS || IS_WINDOWS; + const watchers = new Map(); + + const handleEvent = (dir: string, type: "rename" | "change", filename: string | null) => { + if (!filename) return; // some FS backends emit null filenames; we can't act on those + const full = resolve(dir, filename); + onEvent({ type, path: full }); + // On Linux fallback, a `rename` event on a directory may mean a new + // subdirectory was just created — start watching it too. + if (!useNativeRecursive && type === "rename") { + try { + const stat = statSync(full); + if (stat.isDirectory() && !watchers.has(full)) { + watchDirNonRecursive(full); + } + } catch { + // Path was deleted between event and stat; ignore. + } + } + }; + + const watchDirNonRecursive = (dir: string) => { + if (watchers.has(dir)) return; + try { + const w = watch(dir, (eventType, filename) => { + handleEvent(dir, eventType as "rename" | "change", filename); + }); + w.on("error", () => { + // Directory was deleted or otherwise became unwatchable. Drop it. + watchers.delete(dir); + }); + watchers.set(dir, w); + } catch { + // EACCES, ENOENT, etc. — skip this directory rather than failing the whole watch. + } + }; + + if (useNativeRecursive) { + const w = watch(absRoot, { recursive: true }, (eventType, filename) => { + handleEvent(absRoot, eventType as "rename" | "change", filename); + }); + watchers.set(absRoot, w); + } else { + // Linux: walk the tree at startup, watch each directory. + const stack: string[] = [absRoot]; + while (stack.length > 0) { + const dir = stack.pop()!; + watchDirNonRecursive(dir); + try { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory()) stack.push(join(dir, entry.name)); + } + } catch { + // Permission or vanished entry; skip silently. + } + } + } + + return { + watchedCount() { + return watchers.size; + }, + close() { + for (const w of watchers.values()) { + try { w.close(); } catch {} + } + watchers.clear(); + }, + }; +} diff --git a/src/db.ts b/src/db.ts index 86d0e69..b9610a4 100644 --- a/src/db.ts +++ b/src/db.ts @@ -3,14 +3,18 @@ * * Uses the shared QMD SQLite database. All Smriti tables are prefixed with * `smriti_` to avoid collisions. Does NOT alter existing QMD tables. + * + * DB lifecycle: initSmriti() → createStore() (SDK) → setQmdStore() → initializeSmritiTables() */ import { Database } from "bun:sqlite"; -import * as sqliteVec from "sqlite-vec"; -import { mkdirSync } from "fs"; -import { dirname } from "path"; -import { QMD_DB_PATH } from "./config"; -import { initializeMemoryTables } from "./qmd"; +import { mkdirSync, existsSync, unlinkSync } from "fs"; +import { dirname, join } from "path"; +import { QMD_DB_PATH, SMRITI_SESSIONS_DIR, SMRITI_DIR } from "./config"; +import { initializeMemoryTables, deleteSession, cleanupOrphanedMemoryVectors } from "./qmd"; +import { createStore } from "../qmd/src/index"; +import { setQmdStore, closeQmdStore } from "./store"; +import type { KnowledgeUnit } from "./team/types"; // ============================================================================= // Connection @@ -18,93 +22,16 @@ import { initializeMemoryTables } from "./qmd"; let _db: Database | null = null; -/** Initialize QMD store tables (content, documents, vectors, etc) */ -function initializeQmdStore(db: Database): void { - // Load sqlite-vec extension - sqliteVec.load(db); - db.exec("PRAGMA journal_mode = WAL"); - db.exec("PRAGMA foreign_keys = ON"); - - // Create content-addressable storage - db.exec(` - CREATE TABLE IF NOT EXISTS content ( - hash TEXT PRIMARY KEY, - doc TEXT NOT NULL, - created_at TEXT NOT NULL - ) - `); - - // Documents table - db.exec(` - CREATE TABLE IF NOT EXISTS documents ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - collection TEXT NOT NULL, - path TEXT NOT NULL, - title TEXT NOT NULL, - hash TEXT NOT NULL, - created_at TEXT NOT NULL, - modified_at TEXT NOT NULL, - active INTEGER NOT NULL DEFAULT 1, - FOREIGN KEY (hash) REFERENCES content(hash) ON DELETE CASCADE, - UNIQUE(collection, path) - ) - `); - - // Content vectors - required for vector search - db.exec(` - CREATE TABLE IF NOT EXISTS content_vectors ( - hash TEXT NOT NULL, - seq INTEGER NOT NULL DEFAULT 0, - pos INTEGER NOT NULL DEFAULT 0, - model TEXT NOT NULL, - embedded_at TEXT NOT NULL, - PRIMARY KEY (hash, seq) - ) - `); - - // vectors_vec is managed by QMD at embedding time because dimensions depend on - // the active embedding model. Do not eagerly create it here. - // Migration: older Smriti versions created an incompatible vectors_vec table - // (embedding-only, no hash_seq), which breaks embed/search paths. - try { - const vecTable = db - .prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name='vectors_vec'`) - .get() as { sql: string } | null; - - if (vecTable?.sql && !vecTable.sql.includes("hash_seq")) { - db.exec(`DROP TABLE IF EXISTS vectors_vec`); - } - } catch { - // If sqlite-vec isn't loaded or table introspection fails, continue. - } -} - -/** Get or create the shared database connection */ -export function getDb(path?: string): Database { - if (_db) return _db; - const dbPath = path || QMD_DB_PATH; - // Ensure parent directory exists before creating database file - const dbDir = dirname(dbPath); - if (dbDir !== ".") { - try { - mkdirSync(dbDir, { recursive: true }); - } catch { - // Directory might already exist or be inaccessible (unlikely in normal cases) - } - } - _db = new Database(dbPath); - initializeQmdStore(_db); - // Also initialize QMD memory tables (sessions, messages) - initializeMemoryTables(_db); +/** Return the cached DB connection. Throws if initSmriti() hasn't been called. */ +export function getDb(): Database { + if (!_db) throw new Error("Database not initialized — call initSmriti() first"); return _db; } -/** Close the database connection */ -export function closeDb(): void { - if (_db) { - _db.close(); - _db = null; - } +/** Close the database and release the QMD store (including its LLM backend). */ +export async function closeDb(): Promise { + _db = null; + await closeQmdStore(); } // ============================================================================= @@ -172,6 +99,13 @@ export function initializeSmritiTables(db: Database): void { // Column already exists } + // density_score on smriti_session_meta + try { + db.exec(`ALTER TABLE smriti_session_meta ADD COLUMN density_score REAL DEFAULT 0`); + } catch { + // Column already exists + } + // Migrate smriti_shares if they don't exist (migration) try { db.exec(`ALTER TABLE smriti_shares ADD COLUMN unit_id TEXT`); @@ -271,6 +205,67 @@ export function initializeSmritiTables(db: Database): void { entities TEXT ); + -- Knowledge consolidation: raw Stage-1 extracts, promoted to canonical on reuse + CREATE TABLE IF NOT EXISTS smriti_knowledge_units ( + id TEXT PRIMARY KEY, -- KnowledgeUnit.id (uuid) + session_id TEXT NOT NULL, + project_id TEXT, + topic TEXT NOT NULL, + category TEXT NOT NULL, + relevance REAL NOT NULL DEFAULT 0, -- 0-10, from Stage 1 + entities TEXT, -- JSON array + files TEXT, -- JSON array + plain_text TEXT NOT NULL, -- raw Stage-1 extract + line_ranges TEXT, -- JSON array of {start,end} + content_hash TEXT NOT NULL, -- hashContent({topic,category,plainText}) — Stage-1 dedup key + tier TEXT NOT NULL DEFAULT 'segmented', -- 'segmented' | 'canonical' | 'archived' + retrieval_count INTEGER NOT NULL DEFAULT 0, + last_recalled_at TEXT, + promoted_at TEXT, + canonical_doc_path TEXT, -- relative path under .smriti/knowledge/, set on promotion + share_id TEXT, -- points at the smriti_shares row created on promotion + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + + CREATE INDEX IF NOT EXISTS idx_smriti_knowledge_units_session ON smriti_knowledge_units(session_id); + CREATE INDEX IF NOT EXISTS idx_smriti_knowledge_units_hash ON smriti_knowledge_units(content_hash); + CREATE INDEX IF NOT EXISTS idx_smriti_knowledge_units_tier ON smriti_knowledge_units(tier); + + -- Canonical entity registry: resolves free-text entity mentions (from Stage 1 + -- extraction) onto a stable node, so recurrence is detected regardless of wording. + -- Propagated team/org-wide via .smriti/config.json, same mechanism as custom categories. + CREATE TABLE IF NOT EXISTS smriti_entities ( + id TEXT PRIMARY KEY, -- slug, e.g. "jwt", "redis" + label TEXT NOT NULL, -- canonical display name + entity_type TEXT NOT NULL DEFAULT 'concept', -- 'technology' | 'concept' | 'file' | 'pattern' + aliases TEXT NOT NULL DEFAULT '[]', -- JSON array of raw strings seen + mention_count INTEGER NOT NULL DEFAULT 0, + first_seen_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE INDEX IF NOT EXISTS idx_smriti_entities_label ON smriti_entities(label); + + -- Relationship triples (subject/object polymorphic via type+id, not literal RDF URIs). + -- knowledge_unit -mentions-> entity edges come free from Stage-1 extraction; + -- knowledge_unit -relatesTo/supersedes/contradicts-> knowledge_unit edges are LLM-gated, + -- only at promotion time (see src/learn/consolidate.ts), persisting what + -- ollamaCheckConflicts previously only computed ephemerally. + CREATE TABLE IF NOT EXISTS smriti_relationships ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + subject_type TEXT NOT NULL, -- 'knowledge_unit' | 'entity' | 'session' + subject_id TEXT NOT NULL, + predicate TEXT NOT NULL, -- 'mentions' | 'relatesTo' | 'supersedes' | 'contradicts' + object_type TEXT NOT NULL, + object_id TEXT NOT NULL, + confidence REAL DEFAULT 1.0, + source TEXT DEFAULT 'extraction', -- 'extraction' | 'derived' | 'llm' + created_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE(subject_type, subject_id, predicate, object_type, object_id) + ); + CREATE INDEX IF NOT EXISTS idx_smriti_relationships_subject ON smriti_relationships(subject_type, subject_id); + CREATE INDEX IF NOT EXISTS idx_smriti_relationships_object ON smriti_relationships(object_type, object_id); + CREATE INDEX IF NOT EXISTS idx_smriti_relationships_predicate ON smriti_relationships(predicate); + -- Tool usage tracking CREATE TABLE IF NOT EXISTS smriti_tool_usage ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -447,7 +442,66 @@ export function initializeSmritiTables(db: Database): void { ON smriti_attachments(session_id); CREATE INDEX IF NOT EXISTS idx_smriti_voice_notes_session ON smriti_voice_notes(session_id); + + -- Semantic session clusters (#66) + CREATE TABLE IF NOT EXISTS smriti_session_clusters ( + session_id TEXT NOT NULL, + cluster_id INTEGER NOT NULL, + cluster_name TEXT, + distance REAL, + PRIMARY KEY (session_id, cluster_id) + ); + CREATE INDEX IF NOT EXISTS idx_smriti_session_clusters_cluster + ON smriti_session_clusters(cluster_id); + CREATE INDEX IF NOT EXISTS idx_smriti_session_clusters_name + ON smriti_session_clusters(cluster_name); + + -- Query aliases generated by expandQuery (issue #60) + CREATE TABLE IF NOT EXISTS smriti_session_queries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + session_id TEXT NOT NULL, + query TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'enrich', + created_at TEXT NOT NULL, + UNIQUE(session_id, query) + ); + CREATE INDEX IF NOT EXISTS idx_smriti_session_queries_session + ON smriti_session_queries(session_id); + + CREATE VIRTUAL TABLE IF NOT EXISTS smriti_queries_fts USING fts5( + session_id UNINDEXED, + query, + tokenize='porter unicode61' + ); + + CREATE TRIGGER IF NOT EXISTS smriti_session_queries_ai + AFTER INSERT ON smriti_session_queries + BEGIN + INSERT INTO smriti_queries_fts(rowid, session_id, query) + VALUES (new.id, new.session_id, new.query); + END; + + CREATE TRIGGER IF NOT EXISTS smriti_session_queries_ad + AFTER DELETE ON smriti_session_queries + BEGIN + DELETE FROM smriti_queries_fts WHERE rowid = old.id; + END; `); + + // Prune: 'archived' tier support on smriti_knowledge_units (no CHECK + // constraint on `tier`, so the new value needs no migration — only these + // two nullable columns, set when a canonical unit is archived because a + // `supersedes` edge points at it). + try { + db.exec(`ALTER TABLE smriti_knowledge_units ADD COLUMN archived_at TEXT`); + } catch { + // Column already exists + } + try { + db.exec(`ALTER TABLE smriti_knowledge_units ADD COLUMN archived_reason TEXT`); + } catch { + // Column already exists + } } // ============================================================================= @@ -498,6 +552,12 @@ const DEFAULT_AGENTS = [ log_pattern: null, parser: "claude-web", }, + { + id: "team", + display_name: "Team Import", + log_pattern: null, + parser: "generic", + }, ] as const; /** Default category taxonomy */ @@ -657,14 +717,39 @@ export function migrateFTSToV2(db: Database): void { // Convenience // ============================================================================= -/** Initialize DB, create tables, seed defaults. Returns the DB instance. */ -export function initSmriti(dbPath?: string): Database { - const db = getDb(dbPath); - // getDb() now calls createStore() which initializes QMD tables, - // so we just need to initialize Smriti tables +/** + * Initialize the QMD SDK store, then create all Smriti tables. + * Returns the underlying bun:sqlite Database for Smriti table operations. + */ +export async function initSmriti(dbPath?: string): Promise { + const resolvedPath = dbPath || QMD_DB_PATH; + if (resolvedPath !== ":memory:") { + try { mkdirSync(dirname(resolvedPath), { recursive: true }); } catch { /* exists */ } + } + const store = await createStore({ dbPath: resolvedPath }); + setQmdStore(store); + const db = store.internal.db as unknown as Database; + _db = db; + // busy_timeout is per-connection, not persisted in the DB file — set it on + // every open. Without it, two processes opening the same SQLite file at + // once (e.g. the daemon's flush and a manual `smriti ingest --force`) fail + // immediately with "database is locked" instead of retrying briefly. + db.exec("PRAGMA busy_timeout = 5000"); + initializeMemoryTables(db as any); initializeSmritiTables(db); seedDefaults(db); migrateFTSToV2(db); + + // Register the smriti-sessions QMD collection when the sessions dir exists + if (resolvedPath !== ":memory:" && existsSync(SMRITI_SESSIONS_DIR)) { + try { + await store.addCollection("smriti-sessions", { + path: SMRITI_SESSIONS_DIR, + pattern: "**/*.md", + }); + } catch { /* collection may already exist */ } + } + return db; } @@ -1090,6 +1175,31 @@ export function deleteSidecarRows(db: Database, sessionId: string): void { db.prepare(`DELETE FROM smriti_session_costs WHERE session_id = ?`).run(sessionId); } +/** + * Full sidecar cleanup for a session forget — a superset of deleteSidecarRows + * (which `ingest --force` uses, needing only the narrower tool/file/command/ + * error/cost set that gets re-derived on re-ingest). Also clears + * Smriti-specific metadata/content tables that didn't exist when + * deleteSidecarRows was written. Does NOT touch smriti_knowledge_units or + * smriti_shares — callers (forgetSession) handle those separately since + * canonical (promoted) units are kept unless purging shared knowledge. + */ +export function deleteAllSidecarRows(db: Database, sessionId: string): void { + deleteSidecarRows(db, sessionId); + + db.prepare( + `DELETE FROM smriti_message_tags WHERE message_id IN (SELECT id FROM memory_messages WHERE session_id = ?)` + ).run(sessionId); + db.prepare(`DELETE FROM smriti_session_meta WHERE session_id = ?`).run(sessionId); + db.prepare(`DELETE FROM smriti_session_tags WHERE session_id = ?`).run(sessionId); + db.prepare(`DELETE FROM smriti_artifacts WHERE session_id = ?`).run(sessionId); + db.prepare(`DELETE FROM smriti_thinking WHERE session_id = ?`).run(sessionId); + db.prepare(`DELETE FROM smriti_attachments WHERE session_id = ?`).run(sessionId); + db.prepare(`DELETE FROM smriti_voice_notes WHERE session_id = ?`).run(sessionId); + db.prepare(`DELETE FROM smriti_session_queries WHERE session_id = ?`).run(sessionId); + db.prepare(`DELETE FROM smriti_session_clusters WHERE session_id = ?`).run(sessionId); +} + export function insertGitOperation( db: Database, messageId: number, @@ -1171,3 +1281,539 @@ export function insertVoiceNote( VALUES (?, ?, ?, ?, ?)` ).run(messageId, sessionId, title, transcript, createdAt); } + +// ============================================================================= +// Knowledge Density Scoring (#62) +// ============================================================================= + +export type DensityBreakdown = { + toolCalls: number; + fileWrites: number; + gitOps: number; + decisionTags: number; + errors: number; + totalTokens: number; + score: number; +}; + +/** + * Compute a composite density score (0–1) for a session based on sidecar signals. + * + * Weights: tool calls 25%, file writes 25%, git ops 20%, decision tags 15%, + * errors 10%, token volume 5%. Each signal is linearly capped at a "full" ceiling. + */ +export function computeDensityScore(db: Database, sessionId: string): DensityBreakdown { + const toolCalls = (db.prepare( + `SELECT COUNT(*) as n FROM smriti_tool_usage WHERE session_id = ?` + ).get(sessionId) as { n: number }).n; + + const fileWrites = (db.prepare( + `SELECT COUNT(*) as n FROM smriti_file_operations + WHERE session_id = ? AND operation IN ('write', 'edit', 'create')` + ).get(sessionId) as { n: number }).n; + + const gitOps = (db.prepare( + `SELECT COUNT(*) as n FROM smriti_git_operations WHERE session_id = ?` + ).get(sessionId) as { n: number }).n; + + const decisionTags = (db.prepare( + `SELECT COUNT(*) as n FROM smriti_session_tags + WHERE session_id = ? AND (category_id = 'decision' OR category_id LIKE 'decision/%')` + ).get(sessionId) as { n: number }).n; + + const errors = (db.prepare( + `SELECT COUNT(*) as n FROM smriti_errors WHERE session_id = ?` + ).get(sessionId) as { n: number }).n; + + const totalTokens = (db.prepare( + `SELECT COALESCE(SUM(total_input_tokens + total_output_tokens + total_cache_tokens), 0) as n + FROM smriti_session_costs WHERE session_id = ?` + ).get(sessionId) as { n: number }).n; + + const score = + Math.min(toolCalls / 50, 1) * 0.25 + + Math.min(fileWrites / 20, 1) * 0.25 + + Math.min(gitOps / 10, 1) * 0.20 + + Math.min(decisionTags / 3, 1) * 0.15 + + Math.min(errors / 10, 1) * 0.10 + + Math.min(totalTokens / 200_000, 1) * 0.05; + + return { toolCalls, fileWrites, gitOps, decisionTags, errors, totalTokens, score }; +} + +export function updateDensityScore(db: Database, sessionId: string, score: number): void { + db.prepare( + `UPDATE smriti_session_meta SET density_score = ? WHERE session_id = ?` + ).run(score, sessionId); +} + +export function getDensityScore(db: Database, sessionId: string): number { + const row = db.prepare( + `SELECT density_score FROM smriti_session_meta WHERE session_id = ?` + ).get(sessionId) as { density_score: number } | null; + return row?.density_score ?? 0; +} + +// ============================================================================= +// Knowledge Consolidation (Progressive Summarization) +// ============================================================================= + +export interface StoredKnowledgeUnit { + id: string; + session_id: string; + project_id: string | null; + topic: string; + category: string; + relevance: number; + entities: string[]; + files: string[]; + plain_text: string; + line_ranges: Array<{ start: number; end: number }>; + content_hash: string; + tier: "segmented" | "canonical" | "archived"; + retrieval_count: number; + last_recalled_at: string | null; + promoted_at: string | null; + canonical_doc_path: string | null; + share_id: string | null; + archived_at: string | null; + archived_reason: string | null; +} + +type KnowledgeUnitRow = { + id: string; + session_id: string; + project_id: string | null; + topic: string; + category: string; + relevance: number; + entities: string | null; + files: string | null; + plain_text: string; + line_ranges: string | null; + content_hash: string; + tier: string; + retrieval_count: number; + last_recalled_at: string | null; + promoted_at: string | null; + canonical_doc_path: string | null; + share_id: string | null; + archived_at: string | null; + archived_reason: string | null; +}; + +function deserializeKnowledgeUnit(row: KnowledgeUnitRow): StoredKnowledgeUnit { + return { + ...row, + entities: row.entities ? JSON.parse(row.entities) : [], + files: row.files ? JSON.parse(row.files) : [], + line_ranges: row.line_ranges ? JSON.parse(row.line_ranges) : [], + tier: row.tier as "segmented" | "canonical" | "archived", + }; +} + +/** + * Insert a Stage-1 knowledge unit if its content hash isn't already stored. + * Returns true if inserted, false if it was a duplicate (caller distinguishes + * "stored" from "skipped" the same way shareSegmentedKnowledge does for shares). + */ +export function insertKnowledgeUnit( + db: Database, + unit: KnowledgeUnit, + sessionId: string, + projectId: string | null, + contentHash: string +): boolean { + const exists = db + .prepare(`SELECT 1 FROM smriti_knowledge_units WHERE content_hash = ?`) + .get(contentHash); + if (exists) return false; + + db.prepare( + `INSERT INTO smriti_knowledge_units + (id, session_id, project_id, topic, category, relevance, entities, files, plain_text, line_ranges, content_hash) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + unit.id, + sessionId, + projectId, + unit.topic, + unit.category, + unit.relevance, + JSON.stringify(unit.entities || []), + JSON.stringify(unit.files || []), + unit.plainText, + JSON.stringify(unit.lineRanges || []), + contentHash + ); + return true; +} + +/** Dense sessions (by density_score) that haven't been segmented into knowledge units yet. */ +export function findUnsegmentedDenseSessions( + db: Database, + minDensity: number, + limit?: number +): Array<{ session_id: string; project_id: string | null; density_score: number }> { + const query = ` + SELECT sm.session_id, sm.project_id, sm.density_score + FROM smriti_session_meta sm + WHERE sm.density_score >= ? + AND NOT EXISTS (SELECT 1 FROM smriti_knowledge_units ku WHERE ku.session_id = sm.session_id) + ORDER BY sm.density_score DESC + ${limit ? "LIMIT ?" : ""} + `; + const rows = limit + ? db.prepare(query).all(minDensity, limit) + : db.prepare(query).all(minDensity); + return rows as Array<{ session_id: string; project_id: string | null; density_score: number }>; +} + +/** Segmented units that have proven reuse (via recall) or scored high relevance at extraction time. */ +export function findPromotableUnits( + db: Database, + minRetrievals: number, + minRelevance: number, + minEntityReach?: number +): StoredKnowledgeUnit[] { + // minEntityReach: a unit is promotable if one of its entities is + // independently mentioned by >= minEntityReach OTHER units — a structural + // reuse signal (cross-session recurrence) that doesn't depend on recall() + // ever having been called on this particular unit. + const entityReachClause = minEntityReach + ? `OR id IN ( + SELECT r1.subject_id FROM smriti_relationships r1 + JOIN smriti_relationships r2 + ON r1.object_id = r2.object_id AND r2.predicate = 'mentions' + AND r1.predicate = 'mentions' AND r1.subject_id != r2.subject_id + WHERE r1.subject_type = 'knowledge_unit' + GROUP BY r1.subject_id + HAVING COUNT(DISTINCT r2.subject_id) >= ? + )` + : ""; + const params = minEntityReach + ? [minRetrievals, minRelevance, minEntityReach] + : [minRetrievals, minRelevance]; + + const rows = db + .prepare( + `SELECT * FROM smriti_knowledge_units + WHERE tier = 'segmented' AND (retrieval_count >= ? OR relevance >= ? ${entityReachClause})` + ) + .all(...params) as KnowledgeUnitRow[]; + return rows.map(deserializeKnowledgeUnit); +} + +/** Bump retrieval_count for any knowledge units belonging to a recalled session. No-op if none exist yet. */ +export function incrementRetrievalCount(db: Database, sessionId: string): void { + db.prepare( + `UPDATE smriti_knowledge_units + SET retrieval_count = retrieval_count + 1, + last_recalled_at = datetime('now'), + updated_at = datetime('now') + WHERE session_id = ?` + ).run(sessionId); +} + +export function promoteKnowledgeUnit( + db: Database, + unitId: string, + canonicalDocPath: string, + shareId: string +): void { + db.prepare( + `UPDATE smriti_knowledge_units + SET tier = 'canonical', promoted_at = datetime('now'), + canonical_doc_path = ?, share_id = ?, updated_at = datetime('now') + WHERE id = ?` + ).run(canonicalDocPath, shareId, unitId); +} + +export function listKnowledgeUnits( + db: Database, + options: { tier?: "segmented" | "canonical" | "archived"; minRetrievals?: number; limit?: number } = {} +): StoredKnowledgeUnit[] { + const conditions: string[] = []; + const params: any[] = []; + + if (options.tier) { + conditions.push("tier = ?"); + params.push(options.tier); + } + if (options.minRetrievals !== undefined) { + conditions.push("retrieval_count >= ?"); + params.push(options.minRetrievals); + } + + const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : ""; + const limitClause = options.limit ? "LIMIT ?" : ""; + if (options.limit) params.push(options.limit); + + const rows = db + .prepare( + `SELECT * FROM smriti_knowledge_units ${where} + ORDER BY retrieval_count DESC, relevance DESC + ${limitClause}` + ) + .all(...params) as KnowledgeUnitRow[]; + return rows.map(deserializeKnowledgeUnit); +} + +/** Cascade-delete relationship edges where this knowledge unit is subject or object. */ +function deleteKnowledgeUnitRelationships(db: Database, unitId: string): void { + db.prepare( + `DELETE FROM smriti_relationships WHERE subject_type = 'knowledge_unit' AND subject_id = ?` + ).run(unitId); + db.prepare( + `DELETE FROM smriti_relationships WHERE object_type = 'knowledge_unit' AND object_id = ?` + ).run(unitId); +} + +/** + * Hard-delete a knowledge unit and its relationship edges. Shared by + * forgetSession (removing unpromoted units of a forgotten session) and + * pruneKnowledge (removing stale segmented units) — safe in both cases + * because a 'segmented' unit was never promoted, so nothing external + * (canonical doc, smriti_shares row) references it. + */ +export function deleteKnowledgeUnit(db: Database, unitId: string): void { + deleteKnowledgeUnitRelationships(db, unitId); + db.prepare(`DELETE FROM smriti_knowledge_units WHERE id = ?`).run(unitId); +} + +/** + * Segmented units that failed both promotion paths — the relevance escape + * hatch mirrors findPromotableUnits' own minRelevance, so a unit one + * `consolidate` run away from promoting is never a prune candidate — and are + * old enough that they're unlikely to ever clear the bar. + */ +export function findStaleSegmentedUnits( + db: Database, + maxAgeDays: number, + minRelevance: number +): StoredKnowledgeUnit[] { + const rows = db + .prepare( + `SELECT * FROM smriti_knowledge_units + WHERE tier = 'segmented' AND retrieval_count = 0 AND relevance < ? + AND created_at < datetime('now', '-' || ? || ' days')` + ) + .all(minRelevance, maxAgeDays) as KnowledgeUnitRow[]; + return rows.map(deserializeKnowledgeUnit); +} + +/** Canonical units with an incoming `supersedes` edge (some other unit supersedes them) that aren't already archived. */ +export function findSupersededCanonicalUnits( + db: Database +): Array { + const rows = db + .prepare( + `SELECT ku.*, r.subject_id AS supersededByUnitId, super_ku.topic AS supersededByTopic + FROM smriti_knowledge_units ku + JOIN smriti_relationships r + ON r.object_type = 'knowledge_unit' AND r.object_id = ku.id AND r.predicate = 'supersedes' + JOIN smriti_knowledge_units super_ku ON super_ku.id = r.subject_id + WHERE ku.tier = 'canonical'` + ) + .all() as Array; + return rows.map((r) => ({ ...deserializeKnowledgeUnit(r), supersededByUnitId: r.supersededByUnitId, supersededByTopic: r.supersededByTopic })); +} + +/** Soft-archive a canonical unit — tier -> 'archived', archived_at/reason set. The unit's relationship edges (including the supersedes edge that justified this) are left untouched as the audit trail. */ +export function archiveKnowledgeUnit(db: Database, unitId: string, reason: string): void { + db.prepare( + `UPDATE smriti_knowledge_units + SET tier = 'archived', archived_at = datetime('now'), archived_reason = ?, updated_at = datetime('now') + WHERE id = ?` + ).run(reason, unitId); +} + +// ============================================================================= +// Forget (session deletion) +// ============================================================================= + +export type ForgetOptions = { + /** Permanently delete instead of the default soft delete (active = 0). */ + hard?: boolean; + /** Only meaningful with hard: true. Also delete canonical (promoted) units, their smriti_shares row, and their .smriti/knowledge/*.md doc — normally kept since they've already been shared. */ + purgeShared?: boolean; + /** Where canonical docs live, for purgeShared's file deletion. Defaults to the same convention consolidateKnowledge uses. */ + outputDir?: string; +}; + +export type ForgetResult = { + sessionId: string; + hard: boolean; + unitsDeleted: number; // unpromoted (segmented) knowledge units removed + unitsPurged: number; // canonical units removed, only when purgeShared + canonicalKept: number; // canonical units left in place +}; + +/** + * Forget a session. Soft delete (default) just flips memory_sessions.active + * to 0 — reversible, and already understood by `list --all`/`listSessions`. + * Hard delete removes messages, all sidecar rows, unpromoted knowledge + * units, and orphaned vector embeddings; canonical (promoted) units are kept + * unless purgeShared is set, since they may already be referenced outside + * this session (team sync, a committed .smriti/knowledge/ doc). + */ +export function forgetSession( + db: Database, + sessionId: string, + options: ForgetOptions = {} +): ForgetResult { + const hard = options.hard ?? false; + const purgeShared = options.purgeShared ?? false; + const result: ForgetResult = { + sessionId, + hard, + unitsDeleted: 0, + unitsPurged: 0, + canonicalKept: 0, + }; + + if (!hard) { + deleteSession(db as any, sessionId, false); + return result; + } + + const units = db + .prepare( + `SELECT id, tier, canonical_doc_path FROM smriti_knowledge_units WHERE session_id = ?` + ) + .all(sessionId) as Array<{ id: string; tier: string; canonical_doc_path: string | null }>; + + const outputDir = options.outputDir || join(process.cwd(), SMRITI_DIR); + + for (const u of units) { + if (u.tier !== "canonical") { + deleteKnowledgeUnit(db, u.id); + result.unitsDeleted++; + continue; + } + if (!purgeShared) { + result.canonicalKept++; + continue; + } + deleteKnowledgeUnit(db, u.id); + db.prepare(`DELETE FROM smriti_shares WHERE unit_id = ?`).run(u.id); + if (u.canonical_doc_path) { + try { + unlinkSync(join(outputDir, u.canonical_doc_path)); + } catch { + // Doc already gone or never written under this outputDir — fine. + } + } + result.unitsPurged++; + } + + deleteAllSidecarRows(db, sessionId); + deleteSession(db as any, sessionId, true); + cleanupOrphanedMemoryVectors(db as any); + + return result; +} + +// ============================================================================= +// Session Query Labels (#60) +// ============================================================================= + +export function insertSessionQueries( + db: Database, + sessionId: string, + queries: string[], + source: string = "enrich" +): number { + const now = new Date().toISOString(); + const stmt = db.prepare( + `INSERT OR IGNORE INTO smriti_session_queries(session_id, query, source, created_at) + VALUES (?, ?, ?, ?)` + ); + let inserted = 0; + for (const q of queries) { + const trimmed = q.trim(); + if (trimmed) { + const result = stmt.run(sessionId, trimmed, source, now); + inserted += result.changes; + } + } + return inserted; +} + +export function getSessionQueryCount(db: Database, sessionId: string): number { + return (db.prepare( + `SELECT COUNT(*) as n FROM smriti_session_queries WHERE session_id = ?` + ).get(sessionId) as { n: number }).n; +} + +// ============================================================================= +// QMD Document Index (#59 Phase 4) +// ============================================================================= + +export function getSessionDocPath(sessionId: string): string { + const { join } = require("path"); + const { SMRITI_SESSIONS_DIR: dir } = require("./config"); + return join(dir, `${sessionId}.md`); +} + +export function buildSessionDocument( + sessionId: string, + title: string, + agentId: string | null, + projectId: string | null, + createdAt: string, + messages: { role: string; content: string }[] +): string { + const lines: string[] = [ + `# ${title || sessionId}`, + "", + `agent: ${agentId || "unknown"}`, + `project: ${projectId || "unknown"}`, + `date: ${createdAt.split("T")[0]}`, + `session_id: ${sessionId}`, + "", + ]; + for (const msg of messages) { + lines.push(`**${msg.role}**: ${msg.content}`); + lines.push(""); + } + return lines.join("\n"); +} + +export async function writeSessionDocument( + db: Database, + sessionId: string, + agentId: string | null, + projectId: string | null +): Promise { + const { SMRITI_SESSIONS_DIR: dir } = await import("./config"); + const { mkdirSync: mkDir, writeFileSync } = await import("fs"); + mkDir(dir, { recursive: true }); + + const session = db.prepare( + `SELECT title, created_at FROM memory_sessions WHERE id = ?` + ).get(sessionId) as { title: string; created_at: string } | null; + if (!session) return; + + const messages = db.prepare( + `SELECT role, content FROM memory_messages WHERE session_id = ? ORDER BY created_at ASC` + ).all(sessionId) as { role: string; content: string }[]; + + const content = buildSessionDocument(sessionId, session.title, agentId, projectId, session.created_at, messages); + const { join } = await import("path"); + writeFileSync(join(dir, `${sessionId}.md`), content, "utf-8"); +} + +export function getUnenrichedSessionIds(db: Database, projectId?: string): string[] { + const baseQuery = ` + SELECT sm.session_id + FROM smriti_session_meta sm + LEFT JOIN smriti_session_queries sq ON sq.session_id = sm.session_id + WHERE sq.session_id IS NULL + ${projectId ? "AND sm.project_id = ?" : ""} + `; + const rows = projectId + ? db.prepare(baseQuery).all(projectId) as { session_id: string }[] + : db.prepare(baseQuery).all() as { session_id: string }[]; + return rows.map(r => r.session_id); +} diff --git a/src/digest.ts b/src/digest.ts new file mode 100644 index 0000000..8e4dc20 --- /dev/null +++ b/src/digest.ts @@ -0,0 +1,297 @@ +/** + * digest.ts - Structured work summary for a time window (#63) + * + * Aggregates session activity from sidecar tables into a digest report + * grouped by project. Optionally synthesizes a narrative via Ollama. + */ + +import type { Database } from "bun:sqlite"; +import { ollamaChat } from "./ollama"; + +// ============================================================================= +// Types +// ============================================================================= + +export type DigestSession = { + id: string; + title: string; + projectId: string | null; + agentId: string | null; + toolCount: number; + fileCount: number; + gitCount: number; + errorCount: number; + totalTokens: number; + estimatedCost: number; + densityScore: number; + updatedAt: string; +}; + +export type DigestProject = { + projectId: string | null; + sessionCount: number; + totalTokens: number; + estimatedCost: number; + filesChanged: number; + gitOps: number; + errorCount: number; + topTools: Array<{ toolName: string; count: number }>; + sessions: DigestSession[]; +}; + +export type DigestReport = { + period: { from: string; to: string; days: number }; + totalSessions: number; + totalMessages: number; + totalTokens: number; + estimatedCost: number; + byProject: DigestProject[]; + topErrors: Array<{ message: string; count: number }>; + synthesis?: string; +}; + +// ============================================================================= +// Core +// ============================================================================= + +export async function generateDigest( + db: Database, + options: { + days?: number; + project?: string; + synthesize?: boolean; + model?: string; + maxTokens?: number; + } = {} +): Promise { + const days = options.days ?? 7; + const cutoff = new Date(Date.now() - days * 86_400_000).toISOString(); + const now = new Date().toISOString(); + + // Fetch sessions within the window + let sessionQuery = ` + SELECT + ms.id, + ms.title, + ms.updated_at, + sm.agent_id, + sm.project_id, + COALESCE(sm.density_score, 0) as density_score + FROM memory_sessions ms + JOIN smriti_session_meta sm ON sm.session_id = ms.id + WHERE ms.active = 1 AND ms.updated_at >= ? + `; + const sessionParams: (string | number)[] = [cutoff]; + + if (options.project) { + sessionQuery += ` AND sm.project_id = ?`; + sessionParams.push(options.project); + } + + sessionQuery += ` ORDER BY ms.updated_at DESC`; + + const sessionRows = db.prepare(sessionQuery).all(...sessionParams) as Array<{ + id: string; + title: string; + updated_at: string; + agent_id: string | null; + project_id: string | null; + density_score: number; + }>; + + if (sessionRows.length === 0) { + return { + period: { from: cutoff, to: now, days }, + totalSessions: 0, + totalMessages: 0, + totalTokens: 0, + estimatedCost: 0, + byProject: [], + topErrors: [], + }; + } + + const sessionIds = sessionRows.map((s) => s.id); + const inPlaceholders = sessionIds.map(() => "?").join(","); + + // Total message count + const msgCountRow = db + .prepare(`SELECT COUNT(*) as n FROM memory_messages WHERE session_id IN (${inPlaceholders})`) + .get(...sessionIds) as { n: number }; + + // Tool counts per session + const toolCountRows = db + .prepare( + `SELECT session_id, COUNT(*) as n FROM smriti_tool_usage + WHERE session_id IN (${inPlaceholders}) GROUP BY session_id` + ) + .all(...sessionIds) as { session_id: string; n: number }[]; + const toolCountMap = new Map(toolCountRows.map((r) => [r.session_id, r.n])); + + // File write counts per session + const fileCountRows = db + .prepare( + `SELECT session_id, COUNT(*) as n FROM smriti_file_operations + WHERE session_id IN (${inPlaceholders}) + AND operation IN ('write', 'edit', 'create') + GROUP BY session_id` + ) + .all(...sessionIds) as { session_id: string; n: number }[]; + const fileCountMap = new Map(fileCountRows.map((r) => [r.session_id, r.n])); + + // Git op counts per session + const gitCountRows = db + .prepare( + `SELECT session_id, COUNT(*) as n FROM smriti_git_operations + WHERE session_id IN (${inPlaceholders}) GROUP BY session_id` + ) + .all(...sessionIds) as { session_id: string; n: number }[]; + const gitCountMap = new Map(gitCountRows.map((r) => [r.session_id, r.n])); + + // Error counts per session + const errorCountRows = db + .prepare( + `SELECT session_id, COUNT(*) as n FROM smriti_errors + WHERE session_id IN (${inPlaceholders}) GROUP BY session_id` + ) + .all(...sessionIds) as { session_id: string; n: number }[]; + const errorCountMap = new Map(errorCountRows.map((r) => [r.session_id, r.n])); + + // Costs per session + const costRows = db + .prepare( + `SELECT session_id, + SUM(total_input_tokens + total_output_tokens + total_cache_tokens) as tokens, + SUM(estimated_cost_usd) as cost + FROM smriti_session_costs + WHERE session_id IN (${inPlaceholders}) + GROUP BY session_id` + ) + .all(...sessionIds) as { session_id: string; tokens: number; cost: number }[]; + const costMap = new Map(costRows.map((r) => [r.session_id, r])); + + // Build DigestSession list + const digestSessions: DigestSession[] = sessionRows.map((s) => { + const costs = costMap.get(s.id); + return { + id: s.id, + title: s.title || "(untitled)", + projectId: s.project_id, + agentId: s.agent_id, + toolCount: toolCountMap.get(s.id) ?? 0, + fileCount: fileCountMap.get(s.id) ?? 0, + gitCount: gitCountMap.get(s.id) ?? 0, + errorCount: errorCountMap.get(s.id) ?? 0, + totalTokens: costs?.tokens ?? 0, + estimatedCost: costs?.cost ?? 0, + densityScore: s.density_score, + updatedAt: s.updated_at, + }; + }); + + // Group by project + const projectMap = new Map(); + for (const s of digestSessions) { + const key = s.projectId; + if (!projectMap.has(key)) projectMap.set(key, []); + projectMap.get(key)!.push(s); + } + + // Build per-project tool summaries + const byProject: DigestProject[] = []; + for (const [projectId, sessions] of projectMap) { + const projSessionIds = sessions.map((s) => s.id); + const projPlaceholders = projSessionIds.map(() => "?").join(","); + + const topToolRows = db + .prepare( + `SELECT tool_name, COUNT(*) as n FROM smriti_tool_usage + WHERE session_id IN (${projPlaceholders}) + GROUP BY tool_name ORDER BY n DESC LIMIT 5` + ) + .all(...projSessionIds) as { tool_name: string; n: number }[]; + + byProject.push({ + projectId, + sessionCount: sessions.length, + totalTokens: sessions.reduce((s, r) => s + r.totalTokens, 0), + estimatedCost: sessions.reduce((s, r) => s + r.estimatedCost, 0), + filesChanged: sessions.reduce((s, r) => s + r.fileCount, 0), + gitOps: sessions.reduce((s, r) => s + r.gitCount, 0), + errorCount: sessions.reduce((s, r) => s + r.errorCount, 0), + topTools: topToolRows.map((r) => ({ toolName: r.tool_name, count: r.n })), + sessions, + }); + } + + // Sort projects by activity (token volume descending) + byProject.sort((a, b) => b.totalTokens - a.totalTokens); + + // Top errors across all sessions + const topErrorRows = db + .prepare( + `SELECT message, COUNT(*) as n FROM smriti_errors + WHERE session_id IN (${inPlaceholders}) + GROUP BY message ORDER BY n DESC LIMIT 5` + ) + .all(...sessionIds) as { message: string; n: number }[]; + + const totalTokens = digestSessions.reduce((s, r) => s + r.totalTokens, 0); + const estimatedCost = digestSessions.reduce((s, r) => s + r.estimatedCost, 0); + + const report: DigestReport = { + period: { from: cutoff, to: now, days }, + totalSessions: digestSessions.length, + totalMessages: msgCountRow.n, + totalTokens, + estimatedCost, + byProject, + topErrors: topErrorRows.map((r) => ({ message: r.message, count: r.n })), + }; + + // Optional Ollama synthesis + if (options.synthesize && digestSessions.length > 0) { + const summaryLines: string[] = [ + `Period: last ${days} days`, + `Sessions: ${digestSessions.length}`, + "", + ]; + for (const proj of byProject) { + summaryLines.push(`Project: ${proj.projectId ?? "(no project)"}`); + for (const s of proj.sessions.slice(0, 5)) { + summaryLines.push( + ` - ${s.title} | tools:${s.toolCount} files:${s.fileCount} git:${s.gitCount} errors:${s.errorCount}` + ); + } + if (proj.sessions.length > 5) { + summaryLines.push(` ... and ${proj.sessions.length - 5} more sessions`); + } + } + + try { + const response = await ollamaChat( + [ + { + role: "system", + content: + "You are a work digest generator. Given a summary of recent AI-assisted engineering sessions, " + + "produce a concise narrative (3-5 sentences) describing what was accomplished. " + + "Focus on outcomes: what was built, fixed, or decided. Be specific about projects. " + + "Output only the narrative, no preamble.", + }, + { role: "user", content: summaryLines.join("\n") }, + ], + { + model: options.model, + temperature: 0.3, + maxTokens: options.maxTokens ?? 512, + } + ); + report.synthesis = response.message.content.trim(); + } catch { + // Synthesis is best-effort + } + } + + return report; +} diff --git a/src/format.ts b/src/format.ts index 7c2f332..b32812f 100644 --- a/src/format.ts +++ b/src/format.ts @@ -268,6 +268,123 @@ export function formatShareResult(result: { return lines.join("\n"); } +// ============================================================================= +// Consolidate Result Formatting +// ============================================================================= + +export function formatConsolidateResult(result: { + sessionsSegmented: number; + unitsStored: number; + unitsSkipped: number; + unitsPromoted: number; + unitsPruned?: number; + unitsArchived?: number; + pruneCandidates?: Array<{ id: string; topic: string; tier: string; action: string; reason: string }>; + errors: string[]; +}): string { + const lines = [ + `Sessions segmented: ${result.sessionsSegmented}`, + `Units stored: ${result.unitsStored}`, + `Units skipped (dedup): ${result.unitsSkipped}`, + `Units promoted: ${result.unitsPromoted}`, + ]; + + if (result.pruneCandidates && result.pruneCandidates.length > 0) { + lines.push(""); + lines.push(`Prune candidates (dry-run — rerun with --yes to apply):`); + lines.push( + table( + ["Topic", "Tier", "Action", "Reason"], + result.pruneCandidates.map((c) => [c.topic, c.tier, c.action, c.reason]) + ) + ); + } else if (result.unitsPruned !== undefined || result.unitsArchived !== undefined) { + lines.push(`Units pruned (deleted): ${result.unitsPruned ?? 0}`); + lines.push(`Units archived (superseded): ${result.unitsArchived ?? 0}`); + } + + if (result.errors.length > 0) { + lines.push(`Errors: ${result.errors.length}`); + for (const err of result.errors.slice(0, 5)) { + lines.push(` - ${err}`); + } + } + + return lines.join("\n"); +} + +// ============================================================================= +// Knowledge Units (Learnings) Formatting +// ============================================================================= + +export function formatLearnings( + units: Array<{ + tier: string; + topic: string; + category: string; + retrieval_count: number; + relevance: number; + canonical_doc_path: string | null; + }> +): string { + if (units.length === 0) return "No knowledge units found."; + + const headers = ["Tier", "Topic", "Category", "Retrievals", "Relevance", "Doc Path"]; + const rows = units.map((u) => [ + u.tier === "canonical" ? "✓ canonical" : "segmented", + u.topic, + u.category, + String(u.retrieval_count), + u.relevance.toFixed(1), + u.canonical_doc_path || "-", + ]); + + return table(headers, rows, [14, 40, 20, 10, 9, 40]); +} + +// ============================================================================= +// Entity Graph Formatting (smriti graph ) +// ============================================================================= + +export function formatEntityGraph( + entity: { id: string; label: string; entity_type: string; aliases: string[]; mention_count: number }, + units: Array<{ id: string; topic: string; category: string; relevance: number; tier: string; retrieval_count: number }>, + edges: Array<{ subject_id: string; predicate: string; object_id: string }> +): string { + const lines = [ + `Entity: ${entity.label} (${entity.id})`, + `Type: ${entity.entity_type}`, + `Aliases: ${entity.aliases.join(", ") || "-"}`, + `Mentioned ${entity.mention_count} time(s) across ${units.length} unit(s)`, + "", + ]; + + if (units.length === 0) { + lines.push("No knowledge units mention this entity yet."); + return lines.join("\n"); + } + + const headers = ["Tier", "Topic", "Category", "Retrievals", "Relevance"]; + const rows = units.map((u) => [ + u.tier === "canonical" ? "✓ canonical" : "segmented", + u.topic, + u.category, + String(u.retrieval_count), + u.relevance.toFixed(1), + ]); + lines.push(table(headers, rows, [14, 40, 20, 10, 9])); + + const nonMentionEdges = edges.filter((e) => e.predicate !== "mentions"); + if (nonMentionEdges.length > 0) { + lines.push("", "Relationships between these units:"); + for (const e of nonMentionEdges) { + lines.push(` ${e.subject_id} --${e.predicate}--> ${e.object_id}`); + } + } + + return lines.join("\n"); +} + // ============================================================================= // Sync Result Formatting // ============================================================================= @@ -277,12 +394,20 @@ export function formatSyncResult(result: { imported: number; skipped: number; errors: string[]; + categoriesImported?: number; + entitiesImported?: number; }): string { const lines = [ `Files processed: ${result.filesProcessed}`, `Imported: ${result.imported}`, `Skipped: ${result.skipped}`, ]; + if (result.categoriesImported && result.categoriesImported > 0) { + lines.push(`Categories imported: ${result.categoriesImported}`); + } + if (result.entitiesImported && result.entitiesImported > 0) { + lines.push(`Entities imported: ${result.entitiesImported}`); + } if (result.errors.length > 0) { lines.push(`Errors: ${result.errors.length}`); @@ -375,6 +500,125 @@ export function formatProjectReport( return lines.join("\n"); } +// ============================================================================= +// Density Breakdown Formatting +// ============================================================================= + +export function formatDensityBreakdown(breakdown: { + toolCalls: number; + fileWrites: number; + gitOps: number; + decisionTags: number; + errors: number; + totalTokens: number; + score: number; +}): string { + const bar = (value: number, max: number, width: number = 20): string => { + const filled = Math.round(Math.min(value / max, 1) * width); + return "[" + "=".repeat(filled) + " ".repeat(width - filled) + "]"; + }; + + return [ + `Density Score: ${(breakdown.score * 100).toFixed(1)}%`, + "", + ` Tool calls ${bar(breakdown.toolCalls, 50)} ${breakdown.toolCalls} (cap 50)`, + ` File writes ${bar(breakdown.fileWrites, 20)} ${breakdown.fileWrites} (cap 20)`, + ` Git ops ${bar(breakdown.gitOps, 10)} ${breakdown.gitOps} (cap 10)`, + ` Decisions ${bar(breakdown.decisionTags, 3)} ${breakdown.decisionTags} (cap 3)`, + ` Errors ${bar(breakdown.errors, 10)} ${breakdown.errors} (cap 10)`, + ` Tokens ${bar(breakdown.totalTokens, 200_000)} ${breakdown.totalTokens.toLocaleString()} (cap 200k)`, + ].join("\n"); +} + +// ============================================================================= +// Digest Formatting +// ============================================================================= + +export function formatDigest(report: { + period: { from: string; to: string; days: number }; + totalSessions: number; + totalMessages: number; + totalTokens: number; + estimatedCost: number; + byProject: Array<{ + projectId: string | null; + sessionCount: number; + totalTokens: number; + estimatedCost: number; + filesChanged: number; + gitOps: number; + errorCount: number; + topTools: Array<{ toolName: string; count: number }>; + sessions: Array<{ + id: string; + title: string; + updatedAt: string; + toolCount: number; + fileCount: number; + gitCount: number; + errorCount: number; + densityScore: number; + }>; + }>; + topErrors: Array<{ message: string; count: number }>; + synthesis?: string; +}): string { + const lines: string[] = []; + + const fromDate = report.period.from.slice(0, 10); + const toDate = report.period.to.slice(0, 10); + lines.push(`Digest: ${fromDate} → ${toDate} (${report.period.days}d)`); + lines.push(""); + lines.push(`Sessions: ${report.totalSessions}`); + lines.push(`Messages: ${report.totalMessages.toLocaleString()}`); + lines.push(`Tokens: ${report.totalTokens.toLocaleString()}`); + lines.push(`Est. Cost: $${report.estimatedCost.toFixed(4)}`); + + if (report.synthesis) { + lines.push(""); + lines.push("Summary:"); + for (const line of report.synthesis.split("\n")) { + lines.push(` ${line}`); + } + } + + for (const proj of report.byProject) { + lines.push(""); + lines.push(`Project: ${proj.projectId ?? "(no project)"}`); + lines.push( + ` ${proj.sessionCount} session${proj.sessionCount === 1 ? "" : "s"} | ` + + `${proj.filesChanged} file${proj.filesChanged === 1 ? "" : "s"} | ` + + `${proj.gitOps} git op${proj.gitOps === 1 ? "" : "s"} | ` + + `${proj.errorCount} error${proj.errorCount === 1 ? "" : "s"} | ` + + `$${proj.estimatedCost.toFixed(4)}` + ); + + if (proj.topTools.length > 0) { + const toolStr = proj.topTools.map((t) => `${t.toolName}(${t.count})`).join(" "); + lines.push(` Tools: ${toolStr}`); + } + + for (const s of proj.sessions) { + const density = `${(s.densityScore * 100).toFixed(0)}%`; + const date = s.updatedAt.slice(0, 10); + lines.push( + ` ${s.id.slice(0, 8)} ${pad(s.title, 40)} density:${density} ${date}` + ); + } + } + + if (report.topErrors.length > 0) { + lines.push(""); + lines.push("Top Errors:"); + for (const e of report.topErrors) { + const snippet = e.message?.slice(0, 80) || "(empty)"; + lines.push(` x${e.count} ${snippet}`); + } + } + + return lines.join("\n"); +} + // ============================================================================= // Tag Usage Formatting // ============================================================================= diff --git a/src/index.ts b/src/index.ts index 4e627d0..ab6a2e7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,7 +7,7 @@ * schema-based categorization, and team knowledge sharing. */ -import { initSmriti, closeDb, getCategories, getCategoryTree, addCategory, listProjects, tagSession, getProjectReport, type ProjectInspectReport, getTagUsage, type TagUsageEntry } from "./db"; +import { initSmriti, closeDb, getCategories, getCategoryTree, addCategory, listProjects, tagSession, getProjectReport, type ProjectInspectReport, getTagUsage, type TagUsageEntry, computeDensityScore, updateDensityScore, insertSessionQueries, getUnenrichedSessionIds, listKnowledgeUnits, forgetSession } from "./db"; import { getMessages, getSession, getMemoryStatus, embedMemoryMessages } from "./qmd"; import { ingest, ingestAll } from "./ingest/index"; import { categorizeUncategorized } from "./categorize/classifier"; @@ -16,6 +16,8 @@ import { searchFiltered, listSessions } from "./search/index"; import { recall } from "./search/recall"; import { shareKnowledge } from "./team/share"; import { syncTeamKnowledge, listTeamContributions } from "./team/sync"; +import { consolidateKnowledge } from "./learn/consolidate"; +import { findEntity, getUnitsForEntity, getRelationships } from "./learn/entities"; import { generateContext, compareSessions, @@ -51,8 +53,16 @@ import { formatSyncResult, formatProjectReport, formatTagUsage, + formatDensityBreakdown, + formatDigest, + formatConsolidateResult, + formatLearnings, + formatEntityGraph, json, } from "./format"; +import { generateDigest } from "./digest"; +import { ollamaAsk, ollamaDrift, ollamaCheckConflicts } from "./ollama"; +import { clusterSessions, getClusterSessionIds } from "./cluster"; // ============================================================================= // Arg Parsing Helpers @@ -82,6 +92,109 @@ function getPositional(args: string[], index: number): string | undefined { return undefined; } +// ============================================================================= +// Daemon subcommand dispatch +// ============================================================================= + +async function runDaemonCommand(args: string[]): Promise { + const sub = args[1]; + + if (!sub) { + // Foreground daemon — never returns until SIGTERM / SIGINT. + const { runDaemon } = await import("./daemon"); + const daemon = await runDaemon(); + const watched = daemon.watchedAgents.length > 0 + ? daemon.watchedAgents.join(", ") + : "(none — no agent log dirs found)"; + console.error(`[smriti] daemon started, pid=${daemon.pid}, watching=${watched}`); + // Block forever; server.ts's signal handlers handle shutdown + exit. + await new Promise(() => {}); + return; + } + + if (sub === "install") { + const { installDaemon } = await import("./daemon/install"); + const result = await installDaemon({ force: hasFlag(args, "--force") }); + console.log(`Service file: ${result.servicePath}`); + console.log(` wrote: ${result.wrote}`); + console.log(` already registered: ${result.alreadyRegistered}`); + return; + } + + if (sub === "uninstall") { + const { uninstallDaemon } = await import("./daemon/install"); + const result = await uninstallDaemon(); + console.log(`Service file: ${result.servicePath}`); + console.log(` removed: ${result.removedFile}`); + console.log(` unregistered: ${result.unregistered}`); + return; + } + + if (sub === "status") { + const { getDaemonStatus } = await import("./daemon/client"); + const s = getDaemonStatus(); + if (!s.running) { + console.log("daemon: not running"); + console.log(` PID file: ${s.pidFile}`); + return; + } + console.log("daemon: running"); + console.log(` PID: ${s.pid}`); + if (s.startedAt) { + const uptimeSec = Math.floor((Date.now() - s.startedAt.getTime()) / 1000); + console.log(` started: ${s.startedAt.toISOString()}`); + console.log(` uptime: ${formatUptime(uptimeSec)}`); + } + return; + } + + if (sub === "stop") { + const { stopDaemon } = await import("./daemon/client"); + const r = await stopDaemon(); + if (r.state === "not-running") { + console.log("daemon: not running"); + } else if (r.state === "stopped") { + console.log(`daemon: stopped (PID ${r.pid})`); + } else { + console.log(`daemon: did not exit in time (PID ${r.pid}). Send SIGKILL manually or retry.`); + process.exit(1); + } + return; + } + + if (sub === "logs") { + const { DAEMON_LOG_FILE } = await import("./config"); + const file = Bun.file(DAEMON_LOG_FILE); + if (!(await file.exists())) { + console.error(`No log file at ${DAEMON_LOG_FILE}. Has the daemon ever run?`); + process.exit(1); + } + // tail -F follows the file across rotation, which is what LaunchAgents + // and systemd will do over time. + const proc = Bun.spawn(["tail", "-F", DAEMON_LOG_FILE], { + stdout: "inherit", + stderr: "inherit", + }); + await proc.exited; + return; + } + + console.error(`Unknown daemon subcommand: ${sub}`); + console.error("Usage: smriti daemon [install|uninstall|status|stop|logs]"); + console.error(" smriti daemon (run in foreground)"); + process.exit(1); +} + +function formatUptime(seconds: number): string { + if (seconds < 60) return `${seconds}s`; + const m = Math.floor(seconds / 60); + if (m < 60) return `${m}m ${seconds % 60}s`; + const h = Math.floor(m / 60); + if (h < 24) return `${h}h ${m % 60}m`; + const d = Math.floor(h / 24); + return `${d}d ${h % 24}h`; +} + // ============================================================================= // Commands // ============================================================================= @@ -98,6 +211,8 @@ Commands: recall [options] Smart recall with optional synthesis categorize [options] Auto-categorize sessions tag Manually tag a session + forget [opts] Delete a session (soft by default; --hard --yes for real deletion) + forget --all [filters] Bulk forget, reusing list's --project/--category/--agent filters categories List category tree categories add [opts] Add a custom category tags [options] Show tag usage in sessions @@ -105,6 +220,9 @@ Commands: compare Compare two sessions (tokens, tools, files) compare --last Compare last 2 sessions for current project share [filters] Export knowledge to .smriti/ + consolidate [options] Segment dense sessions, promote reused units, prune stale/superseded ones + learnings [options] List extracted knowledge units (tier, retrievals, relevance) + graph Show a canonical entity's mentions and relationship edges sync Import team knowledge from .smriti/ team View team contributions list [filters] List sessions @@ -113,6 +231,14 @@ Commands: projects [id] List projects or inspect a project insights [subcommand] Cost & usage analysis dashboard embed Embed new messages for vector search + enrich [--density] [--queries] [--clusters] Compute/update density scores, query labels, or clusters + ask Answer a question from work history (RAG) + drift Show how thinking on a topic evolved over time + clusters [options] Discover topic clusters from session embeddings + digest [options] Show work digest for a time window + config show Show current .smriti/config.json + config add-category Add a custom category to DB and team config + daemon [subcommand] Cross-agent capture daemon (see Daemon options) upgrade Update smriti to the latest version help Show this help @@ -122,6 +248,12 @@ Filters (apply to search, recall, list, share): --agent Filter by agent --limit Max results (default varies by command) +Forget options: + --hard Permanently delete instead of soft delete (requires --yes) + --yes Confirm --hard (required — no confirmation prompt otherwise) + --purge-shared With --hard, also delete canonical (promoted) units, their + smriti_shares row, and their .smriti/knowledge/*.md doc + Ingest options: smriti ingest claude Ingest Claude Code sessions smriti ingest claude-web Claude.ai data export @@ -129,7 +261,8 @@ Ingest options: smriti ingest codex Ingest Codex CLI sessions smriti ingest cline Ingest Cline CLI sessions smriti ingest copilot Ingest GitHub Copilot (VS Code) sessions - smriti ingest cursor --project-path + smriti ingest cursor Ingest Cursor sessions (all workspaces) + smriti ingest cursor --project-path Filter to a specific project path smriti ingest file [--format chat|jsonl] [--title ] [--whole] smriti ingest all Ingest from all known agents (claude, codex, cline, copilot) --force Re-ingest sessions (delete sidecar data, re-extract) @@ -145,6 +278,10 @@ Recall options: --synthesize Synthesize results via Ollama --model Ollama model for synthesis --max-tokens Max synthesis tokens + --fast Skip query expansion and reranking + --wide Search all projects (rerank with current project as intent) + --check-conflicts Detect contradictions among recall results (opt-in, uses Ollama) + --cluster Filter recall to sessions in a named cluster Context options: --project Project filter (auto-detect from cwd) @@ -159,6 +296,11 @@ Share options: --segmented Use 3-stage segmentation pipeline (beta) --min-relevance Relevance threshold for segmented mode (default: 6) +Consolidate options: + --prune Also run the prune phase (dry-run by default — prints candidates, deletes nothing) + --yes, --apply Actually delete/archive prune candidates (requires --prune) + --prune-stale-days Age threshold for stale segmented units (default: 30) + Insights options: smriti insights Full dashboard smriti insights session Session deep dive @@ -167,17 +309,44 @@ Insights options: smriti insights errors [--project ] Error analysis smriti insights tools [--project ] Tool reliability +Daemon options: + smriti daemon Run daemon in foreground (debugging) + smriti daemon install [--force] Install LaunchAgent (macOS) or systemd unit (Linux) + smriti daemon uninstall Reverse install — stop daemon, remove service file + smriti daemon status Show PID, uptime, watched agents + smriti daemon stop Send SIGTERM to the running daemon + smriti daemon logs Tail the daemon log file + Examples: smriti ingest claude smriti ingest copilot smriti search "auth" --project myapp smriti recall "how did we set up auth" --synthesize smriti categorize + smriti consolidate + smriti consolidate --prune + smriti consolidate --prune --yes smriti list --category decision --project myapp smriti share --category decision smriti sync smriti insights --json + smriti enrich --density + smriti enrich --queries + smriti enrich --queries --project myapp --dry-run + smriti digest + smriti digest --days 14 --project myapp --synthesize smriti upgrade + +Enrich options: + --density Recompute density scores for all sessions + --queries Generate search aliases via LLM query expansion + --dry-run Print what would be generated, don't write + +Digest options: + --days Lookback window in days (default: 7) + --project Filter to a specific project + --synthesize Generate narrative summary via Ollama + --model Ollama model for synthesis `; async function main() { @@ -196,8 +365,15 @@ async function main() { return; } + // Daemon subcommands — handled before initSmriti() because the foreground + // daemon opens its own DB handle per flush (smoke-test finding 3). + if (command === "daemon") { + await runDaemonCommand(args); + return; + } + // Initialize DB - const db = initSmriti(); + const db = await initSmriti(); try { switch (command) { @@ -290,9 +466,20 @@ async function main() { process.exit(1); } + const recallProject = getArg(args, "--project"); + const wideMode = hasFlag(args, "--wide"); + const clusterFilter = getArg(args, "--cluster"); + const clusterSessionIds = clusterFilter ? getClusterSessionIds(db, clusterFilter) : null; + + if (clusterFilter && clusterSessionIds !== null && clusterSessionIds.length === 0) { + console.error(`No sessions found for cluster: ${clusterFilter}`); + console.error("Run 'smriti clusters' to see available clusters."); + process.exit(1); + } + const result = await recall(db, query, { category: getArg(args, "--category"), - project: getArg(args, "--project"), + project: recallProject || undefined, agent: getArg(args, "--agent"), limit: Number(getArg(args, "--limit")) || undefined, synthesize: hasFlag(args, "--synthesize"), @@ -302,20 +489,142 @@ async function main() { includeArtifacts: !hasFlag(args, "--no-artifacts"), includeAttachments: !hasFlag(args, "--no-attachments"), includeVoiceNotes: !hasFlag(args, "--no-voice-notes"), + fast: hasFlag(args, "--fast"), + wide: wideMode, }); + // Apply --cluster filter: keep only sessions belonging to the cluster + if (clusterSessionIds && clusterSessionIds.length > 0) { + const clusterSet = new Set(clusterSessionIds); + result.results = result.results.filter(r => clusterSet.has(r.session_id)); + } + + const checkConflicts = hasFlag(args, "--check-conflicts"); + + // In --wide mode, look up project info for cross-project badge + if (wideMode && recallProject && result.results.length > 0) { + const sessionIds = result.results.map(r => r.session_id); + const placeholders = sessionIds.map(() => "?").join(","); + const projRows = db.prepare( + `SELECT session_id, project_id FROM smriti_session_meta WHERE session_id IN (${placeholders})` + ).all(...sessionIds) as { session_id: string; project_id: string }[]; + const projMap = new Map(projRows.map(r => [r.session_id, r.project_id])); + for (const r of result.results) { + const proj = projMap.get(r.session_id); + if (proj && proj !== recallProject && !(r as any).project) { + (r as any).project = proj; + } + } + } + + // Contradiction detection (opt-in) + let conflicts: { pair: [number, number]; description: string }[] = []; + if (checkConflicts && result.results.length >= 2) { + const passages = result.results.slice(0, 5).map((r, i) => ({ + n: i + 1, + title: r.session_title || r.session_id, + content: r.content, + })); + try { + conflicts = await ollamaCheckConflicts(query, passages); + } catch { + // Ollama unavailable — skip conflict detection + } + } + if (hasFlag(args, "--json")) { - console.log(json(result)); + console.log(json({ ...result, conflicts })); } else { console.log(formatSearchResults(result.results)); if (result.synthesis) { console.log("\n--- Synthesis ---\n"); console.log(result.synthesis); } + if (conflicts.length > 0) { + console.log("\n⚠ Conflicts detected:"); + for (const c of conflicts) { + const a = result.results[c.pair[0] - 1]; + const b = result.results[c.pair[1] - 1]; + console.log(` [${c.pair[0]}] vs [${c.pair[1]}]: ${c.description}`); + if (a && b) { + console.log(` ${a.session_id} — ${a.session_title || "(untitled)"}`); + console.log(` ${b.session_id} — ${b.session_title || "(untitled)"}`); + } + } + } } break; } + // ===================================================================== + // ASK (RAG question-answering) + // ===================================================================== + case "ask": { + const question = args[1]; + if (!question) { + console.error('Usage: smriti ask "" [options]'); + process.exit(1); + } + + const noSynthesize = hasFlag(args, "--no-synthesize"); + const askLimit = Number(getArg(args, "--limit")) || 5; + const askModel = getArg(args, "--model"); + const askProject = getArg(args, "--project"); + const askAgent = getArg(args, "--agent"); + + // Multi-angle recall (expandQuery + rerank already default-on) + const askResult = await recall(db, question, { + limit: askLimit, + synthesize: false, + project: askProject || undefined, + agent: askAgent || undefined, + fast: false, + }); + + if (hasFlag(args, "--json")) { + const sources = askResult.results.map((r, i) => ({ + n: i + 1, + session_id: r.session_id, + session_title: r.session_title, + score: r.score, + content: r.content, + })); + console.log(json({ question, sources })); + break; + } + + if (noSynthesize || askResult.results.length === 0) { + console.log(formatSearchResults(askResult.results)); + break; + } + + // Format sources for Ollama + const sourcesText = askResult.results + .map((r, i) => `[${i + 1}] ${r.session_title || r.session_id}\n${r.content}`) + .join("\n\n---\n\n"); + + let answer: string | undefined; + try { + answer = await ollamaAsk(question, sourcesText, { model: askModel || undefined }); + } catch { + answer = undefined; + } + + if (answer) { + console.log(answer); + console.log("\nSources:"); + askResult.results.forEach((r, i) => { + const date = r.session_id ? new Date(r.session_id).toLocaleDateString("en-US", { month: "short", day: "numeric" }) : ""; + console.log(` [${i + 1}] ${r.session_id} — ${r.session_title || "(untitled)"}${date ? ` (${date})` : ""}`); + }); + } else { + console.log("(Ollama unavailable — returning sources)\n"); + console.log(formatSearchResults(askResult.results)); + } + + break; + } + // ===================================================================== // CATEGORIZE // ===================================================================== @@ -357,6 +666,61 @@ async function main() { break; } + // ===================================================================== + // FORGET + // ===================================================================== + case "forget": { + const all = hasFlag(args, "--all"); + const sessionId = getPositional(args, 1); + if (!sessionId && !all) { + console.error("Usage: smriti forget [--hard] [--yes] [--purge-shared]"); + console.error(" smriti forget --all [--project ] [--category ] [--agent ] [--hard] [--yes] [--purge-shared]"); + process.exit(1); + } + + const hard = hasFlag(args, "--hard"); + const purgeShared = hasFlag(args, "--purge-shared"); + if (hard && !hasFlag(args, "--yes")) { + console.error("--hard permanently deletes session data. Re-run with --yes to confirm."); + process.exit(1); + } + + const targetIds = all + ? listSessions(db, { + project: getArg(args, "--project"), + category: getArg(args, "--category"), + agent: getArg(args, "--agent"), + includeInactive: true, + }).map((s) => s.id) + : [sessionId!]; + + if (targetIds.length === 0) { + console.log("No matching sessions to forget."); + break; + } + + let deleted = 0; + let purged = 0; + let kept = 0; + for (const id of targetIds) { + const r = forgetSession(db, id, { hard, purgeShared }); + deleted += r.unitsDeleted; + purged += r.unitsPurged; + kept += r.canonicalKept; + } + + console.log(`Forgot ${targetIds.length} session(s) (${hard ? "hard delete" : "soft delete"}).`); + if (hard) { + console.log(` Unpromoted knowledge units removed: ${deleted}`); + if (purgeShared) { + console.log(` Canonical knowledge units purged: ${purged}`); + } else if (kept > 0) { + console.log(` Canonical knowledge units kept (already shared — pass --purge-shared to also remove): ${kept}`); + } + } + break; + } + // ===================================================================== // CATEGORIES // ===================================================================== @@ -527,6 +891,82 @@ async function main() { break; } + // ===================================================================== + // CONSOLIDATE + // ===================================================================== + case "consolidate": { + const prune = hasFlag(args, "--prune"); + const pruneApply = hasFlag(args, "--yes") || hasFlag(args, "--apply"); + const result = await consolidateKnowledge(db, { + minDensity: Number(getArg(args, "--min-density")) || undefined, + minRetrievals: Number(getArg(args, "--min-retrievals")) || undefined, + minRelevance: Number(getArg(args, "--min-relevance")) || undefined, + minEntityReach: Number(getArg(args, "--min-entity-reach")) || undefined, + model: getArg(args, "--model"), + outputDir: getArg(args, "--output"), + sessionLimit: Number(getArg(args, "--session-limit")) || undefined, + prune, + pruneStaleDays: Number(getArg(args, "--prune-stale-days")) || undefined, + pruneApply, + onProgress: (msg) => console.log(` ${msg}`), + }); + + console.log(formatConsolidateResult(result)); + if (prune && !pruneApply && result.pruneCandidates && result.pruneCandidates.length > 0) { + console.log("\nRun again with --prune --yes to apply."); + } + break; + } + + // ===================================================================== + // LEARNINGS + // ===================================================================== + case "learnings": { + const units = listKnowledgeUnits(db, { + tier: getArg(args, "--tier") as "segmented" | "canonical" | undefined, + minRetrievals: Number(getArg(args, "--min-retrievals")) || undefined, + limit: Number(getArg(args, "--limit")) || 50, + }); + + if (hasFlag(args, "--json")) { + console.log(json(units)); + } else { + console.log(formatLearnings(units)); + } + break; + } + + // ===================================================================== + // GRAPH + // ===================================================================== + case "graph": { + const query = getPositional(args, 1); + if (!query) { + console.error("Usage: smriti graph "); + process.exit(1); + } + + const entity = findEntity(db, query); + if (!entity) { + console.log(`No entity found matching "${query}".`); + break; + } + + const units = getUnitsForEntity(db, entity.id); + const unitIds = new Set(units.map((u) => u.id)); + const edges = units.flatMap((u) => + getRelationships(db, { subjectType: "knowledge_unit", subjectId: u.id }) + .filter((r) => r.predicate !== "mentions" && unitIds.has(r.object_id)) + ); + + if (hasFlag(args, "--json")) { + console.log(json({ entity, units, edges })); + } else { + console.log(formatEntityGraph(entity, units, edges)); + } + break; + } + // ===================================================================== // SYNC // ===================================================================== @@ -881,6 +1321,339 @@ async function main() { break; } + // ===================================================================== + // ENRICH + // ===================================================================== + case "enrich": { + const density = hasFlag(args, "--density"); + const queries = hasFlag(args, "--queries"); + const clusters = hasFlag(args, "--clusters"); + const sessionFilter = getArg(args, "--session"); + const projectFilter = getArg(args, "--project"); + const dryRun = hasFlag(args, "--dry-run"); + + if (!density && !queries && !clusters) { + console.error("Usage: smriti enrich --density | --queries | --clusters [--session ] [--project ] [--dry-run]"); + process.exit(1); + } + + if (density) { + // Backfill density scores for all (or one) session + let sessionIds: string[]; + if (sessionFilter) { + sessionIds = [sessionFilter]; + } else { + sessionIds = ( + db.prepare(`SELECT session_id FROM smriti_session_meta`).all() as { session_id: string }[] + ).map((r) => r.session_id); + } + + console.log(`Computing density scores for ${sessionIds.length} session${sessionIds.length === 1 ? "" : "s"}...`); + let updated = 0; + for (const sid of sessionIds) { + const breakdown = computeDensityScore(db, sid); + updateDensityScore(db, sid, breakdown.score); + updated++; + if (sessionFilter) { + console.log(formatDensityBreakdown(breakdown)); + } + } + if (!sessionFilter) { + console.log(`Updated ${updated} density scores.`); + } + } + + if (queries) { + const { getQmdStore } = await import("./store"); + const sessionIds = sessionFilter + ? [sessionFilter] + : getUnenrichedSessionIds(db, projectFilter || undefined); + + console.log(`Enriching ${sessionIds.length} session${sessionIds.length === 1 ? "" : "s"} with query labels...`); + let enriched = 0; + let skipped = 0; + + for (let i = 0; i < sessionIds.length; i++) { + const sid = sessionIds[i]!; + const session = db.prepare(`SELECT title, summary FROM memory_sessions WHERE id = ?`).get(sid) as { title: string; summary: string | null } | null; + if (!session?.title) { skipped++; continue; } + + const input = session.title + (session.summary ? ". " + session.summary : ""); + process.stdout.write(` [${i + 1}/${sessionIds.length}] ${session.title.slice(0, 60)}...`); + + try { + const store = getQmdStore(); + const expanded = await store.internal.expandQuery(input); + const queryTexts = expanded.map(e => e.query).filter(Boolean); + + if (dryRun) { + console.log(`\n → ${queryTexts.join(" | ")}`); + } else { + const n = insertSessionQueries(db, sid, queryTexts); + process.stdout.write(` +${n}\n`); + enriched++; + } + } catch { + process.stdout.write(` (LLM unavailable, skipped)\n`); + skipped++; + } + } + + if (!dryRun) { + console.log(`\nEnriched ${enriched} sessions${skipped > 0 ? `, skipped ${skipped}` : ""}.`); + } + } + + if (clusters) { + const k = Number(getArg(args, "--k")) || undefined; + const model = getArg(args, "--model"); + console.log("Clustering sessions..."); + const clusterResult = await clusterSessions(db as any, { + projectId: projectFilter || undefined, + k, + model, + }); + if (clusterResult.clusters.length === 0) { + console.log("Not enough sessions with embeddings to cluster. Run 'smriti embed' first."); + } else { + for (const c of clusterResult.clusters) { + const lastActive = c.lastActive ? new Date(c.lastActive).toLocaleDateString("en-US", { month: "short", day: "numeric", year: "numeric" }) : ""; + console.log(` ${c.name.padEnd(40)} ${c.sessionIds.length} session${c.sessionIds.length === 1 ? "" : "s"}${lastActive ? ` (${lastActive})` : ""}`); + } + console.log(`\n${clusterResult.clusters.length} clusters across ${clusterResult.totalSessions} sessions.`); + } + } + + break; + } + + // ===================================================================== + // CLUSTERS + // ===================================================================== + case "clusters": { + const k = Number(getArg(args, "--k")) || undefined; + const model = getArg(args, "--model"); + const projectId = getArg(args, "--project"); + + console.log("Clustering sessions..."); + const clusterResult = await clusterSessions(db as any, { projectId, k, model }); + + if (clusterResult.clusters.length === 0) { + console.log("Not enough sessions with embeddings to cluster."); + console.log("Run 'smriti embed' first to build embeddings, then re-run."); + break; + } + + if (hasFlag(args, "--json")) { + console.log(json(clusterResult)); + break; + } + + console.log(`\n${clusterResult.clusters.length} clusters across ${clusterResult.totalSessions} sessions\n`); + for (const c of clusterResult.clusters) { + const lastActive = c.lastActive ? new Date(c.lastActive).toLocaleDateString("en-US", { month: "short", day: "numeric", year: "numeric" }) : ""; + console.log(` ${c.name}`); + console.log(` ${c.sessionIds.length} session${c.sessionIds.length === 1 ? "" : "s"}${lastActive ? ` · last active ${lastActive}` : ""}`); + } + break; + } + + // ===================================================================== + // DRIFT (temporal evolution) + // ===================================================================== + case "drift": { + const driftTopic = args[1]; + if (!driftTopic) { + console.error('Usage: smriti drift "" [options]'); + process.exit(1); + } + + const driftProject = getArg(args, "--project"); + const driftSince = getArg(args, "--since"); + const driftLimit = Number(getArg(args, "--limit")) || 10; + const noSynthesizeDrift = hasFlag(args, "--no-synthesize"); + + // Recall all matching sessions (high limit, no session dedup — we want all mentions) + const driftResult = await recall(db, driftTopic, { + limit: driftLimit * 2, + synthesize: false, + project: driftProject || undefined, + fast: false, + }); + + if (driftResult.results.length < 2) { + console.log("Not enough history to show evolution."); + if (driftResult.results.length === 1) { + console.log(formatSearchResults(driftResult.results)); + } + break; + } + + // Enrich with session dates from memory_sessions + const sessionIds = [...new Set(driftResult.results.map(r => r.session_id))]; + const placeholders = sessionIds.map(() => "?").join(","); + const dateRows = db.prepare( + `SELECT id, created_at, updated_at FROM memory_sessions WHERE id IN (${placeholders})` + ).all(...sessionIds) as { id: string; created_at: string; updated_at: string }[]; + const dateMap = new Map(dateRows.map(r => [r.id, r])); + + // Filter by --since if given + let filteredResults = driftResult.results; + if (driftSince) { + const sinceDate = new Date(driftSince).getTime(); + filteredResults = filteredResults.filter(r => { + const d = dateMap.get(r.session_id); + return d ? new Date(d.created_at).getTime() >= sinceDate : true; + }); + } + + // Deduplicate by session and sort chronologically + const seenSessions = new Set(); + const chronological = filteredResults + .filter(r => { + if (seenSessions.has(r.session_id)) return false; + seenSessions.add(r.session_id); + return true; + }) + .sort((a, b) => { + const da = dateMap.get(a.session_id)?.created_at ?? ""; + const db2 = dateMap.get(b.session_id)?.created_at ?? ""; + return da.localeCompare(db2); + }) + .slice(0, driftLimit); + + if (hasFlag(args, "--json")) { + const timeline = chronological.map((r, i) => ({ + n: i + 1, + session_id: r.session_id, + session_title: r.session_title, + date: dateMap.get(r.session_id)?.created_at, + content: r.content, + })); + console.log(json({ topic: driftTopic, timeline })); + break; + } + + console.log(`\n${driftTopic} — evolution across ${chronological.length} session${chronological.length === 1 ? "" : "s"}\n`); + const timelineText = chronological.map(r => { + const d = dateMap.get(r.session_id); + const date = d ? new Date(d.created_at).toLocaleDateString("en-US", { month: "short", day: "numeric" }) : "?"; + const proj = (r as any).project ? ` [${(r as any).project}]` : (driftProject ? ` [${driftProject}]` : ""); + return `${date}${proj} ${r.session_title || r.session_id}\n ${r.content.slice(0, 200)}`; + }).join("\n\n"); + + console.log(timelineText); + + if (!noSynthesizeDrift) { + try { + const narrative = await ollamaDrift(driftTopic, timelineText); + console.log("\n--- Evolution narrative ---\n"); + console.log(narrative); + } catch { + // Ollama unavailable — timeline shown above is the fallback + } + } + + break; + } + + // ===================================================================== + // DIGEST + // ===================================================================== + case "digest": { + const days = Number(getArg(args, "--days")) || 7; + const project = getArg(args, "--project"); + const synthesize = hasFlag(args, "--synthesize"); + const model = getArg(args, "--model"); + + const report = await generateDigest(db, { + days, + project, + synthesize, + model, + }); + + if (hasFlag(args, "--json")) { + console.log(json(report)); + } else { + console.log(formatDigest(report)); + } + break; + } + + // ===================================================================== + // CONFIG (team config.json management) + // ===================================================================== + case "config": { + const { readConfig, writeConfig, exportCustomCategories } = await import("./team/config"); + const sub = args[1]; + const smritiDir = (() => { + const project = getArg(args, "--project"); + if (project) { + const p = db.prepare(`SELECT path FROM smriti_projects WHERE id = ?`).get(project) as { path: string } | null; + if (p?.path) return require("path").join(p.path, ".smriti"); + } + return require("path").join(process.cwd(), ".smriti"); + })(); + + if (!sub || sub === "show") { + const config = readConfig(smritiDir); + if (hasFlag(args, "--json")) { + console.log(json(config)); + } else { + console.log(`Config: ${smritiDir}/config.json`); + console.log(` version: ${config.version}`); + const cats = config.categories ?? []; + if (cats.length > 0) { + console.log(` custom categories (${cats.length}):`); + for (const c of cats) { + console.log(` ${c.id}${c.parent ? ` (parent: ${c.parent})` : ""} — ${c.name}`); + } + } else { + console.log(" custom categories: none"); + } + } + } else if (sub === "add-category") { + const id = args[2]; + const name = getArg(args, "--name"); + if (!id || !name) { + console.error("Usage: smriti config add-category --name [--parent ] [--description ] [--project ]"); + process.exit(1); + } + const parent = getArg(args, "--parent"); + const description = getArg(args, "--description"); + + // Add to local DB + const { addCategory } = await import("./db"); + addCategory(db, id, name, parent, description); + console.log(`Added category: ${id} (${name})`); + + // Write to config.json + const { mkdirSync } = await import("fs"); + mkdirSync(smritiDir, { recursive: true }); + const config = readConfig(smritiDir); + const categories = config.categories ?? []; + if (!categories.find(c => c.id === id)) { + categories.push({ id, name, ...(parent ? { parent } : {}), ...(description ? { description } : {}) }); + } + await writeConfig(smritiDir, { ...config, version: 2, categories }); + console.log(`Written to ${smritiDir}/config.json`); + } else if (sub === "sync-categories") { + // Export current DB custom categories into config.json + const { mkdirSync } = await import("fs"); + mkdirSync(smritiDir, { recursive: true }); + const config = readConfig(smritiDir); + const categories = exportCustomCategories(db); + await writeConfig(smritiDir, { ...config, version: categories.length > 0 ? 2 : config.version, categories }); + console.log(`Synced ${categories.length} custom category${categories.length === 1 ? "" : "ies"} to ${smritiDir}/config.json`); + } else { + console.error(`Unknown config subcommand: ${sub}`); + console.error("Usage: smriti config show | add-category | sync-categories"); + process.exit(1); + } + break; + } + // ===================================================================== // UNKNOWN // ===================================================================== @@ -890,7 +1663,7 @@ async function main() { process.exit(1); } } finally { - closeDb(); + await closeDb(); } } diff --git a/src/ingest/codex.ts b/src/ingest/codex.ts index 8be7a80..b677901 100644 --- a/src/ingest/codex.ts +++ b/src/ingest/codex.ts @@ -17,8 +17,16 @@ type CodexEntry = { content?: string | Array<{ type: string; text?: string }>; timestamp?: string; session_id?: string; + // Rollout format (codex_cli_rs >= ~0.40): messages wrapped in payload + payload?: { + type?: string; + role?: string; + content?: string | Array<{ type: string; text?: string }>; + }; }; +const TEXT_BLOCK_TYPES = new Set(["text", "input_text", "output_text"]); + function extractContent( content: string | Array<{ type: string; text?: string }> | undefined ): string { @@ -26,13 +34,23 @@ function extractContent( if (typeof content === "string") return content; if (Array.isArray(content)) { return content - .filter((b) => b.type === "text" && b.text) + .filter((b) => TEXT_BLOCK_TYPES.has(b.type) && b.text) .map((b) => b.text!) .join("\n"); } return ""; } +/** Injected context blocks Codex records as user messages — not real conversation */ +function isInjectedContext(text: string): boolean { + return ( + text.startsWith("# AGENTS.md instructions") || + text.startsWith("") || + text.startsWith("") || + text.startsWith("") + ); +} + /** * Parse a single Codex JSONL file into normalized messages. */ @@ -48,11 +66,21 @@ export function parseCodexJsonl(content: string): ParsedMessage[] { continue; } - const role = entry.role || entry.type; + // Rollout format: unwrap response_item message payloads + let role: string | undefined; + let content: CodexEntry["content"]; + if (entry.type === "response_item" && entry.payload?.type === "message") { + role = entry.payload.role; + content = entry.payload.content; + } else { + role = entry.role || entry.type; + content = entry.content; + } if (!role || (role !== "user" && role !== "assistant")) continue; - const text = extractContent(entry.content); + const text = extractContent(content); if (!text.trim()) continue; + if (role === "user" && isInjectedContext(text)) continue; messages.push({ role, diff --git a/src/ingest/copilot.ts b/src/ingest/copilot.ts index 5a171f7..68c2c67 100644 --- a/src/ingest/copilot.ts +++ b/src/ingest/copilot.ts @@ -28,7 +28,11 @@ type CopilotTurn = { role?: string; content?: string; /** Some versions nest content here */ - message?: string | { value?: string }; + message?: string | { value?: string; text?: string }; + /** Response items (VS Code 1.90+): markdown text lives in `value` */ + value?: string; + /** Response item kind: markdownContent, thinking, progressTaskSerialized, ... */ + kind?: string; timestamp?: string | number; }; @@ -119,11 +123,29 @@ export function deriveProjectId(workspacePath: string): string { // Parsing // ============================================================================= +/** Response item kinds that carry no user-facing conversation text */ +const NON_TEXT_RESPONSE_KINDS = new Set([ + "thinking", + "progressTaskSerialized", + "mcpServersStarting", + "toolInvocationSerialized", + "prepareToolInvocation", + "codeblockUri", + "undoStop", +]); + /** Extract text from a turn regardless of which VS Code version wrote it */ function extractTurnText(turn: CopilotTurn): string { if (typeof turn.content === "string") return turn.content; if (typeof turn.message === "string") return turn.message; - if (typeof turn.message === "object" && turn.message?.value) return turn.message.value; + if (typeof turn.message === "object") { + if (turn.message?.value) return turn.message.value; + if (turn.message?.text) return turn.message.text; + } + // Response items: markdown text in `value` (skip thinking/progress/tool noise) + if (typeof turn.value === "string" && !NON_TEXT_RESPONSE_KINDS.has(turn.kind ?? "")) { + return turn.value; + } return ""; } @@ -142,9 +164,28 @@ export function parseCopilotJson(content: string): ParsedMessage[] { let session: CopilotSession; try { - session = JSON.parse(content); + const parsed = JSON.parse(content); + // Single-line JSONL snapshot: {kind: 0, v: {...session...}} + session = parsed?.kind === 0 && parsed.v && typeof parsed.v === "object" + ? (parsed.v as CopilotSession) + : parsed; } catch { - return messages; + // JSONL format (VS Code 1.10x+): lines of {kind, v}; kind 0 carries the + // full session snapshot — use the last snapshot in the file. + let snapshot: CopilotSession | null = null; + for (const line of content.split("\n")) { + if (!line.trim()) continue; + try { + const entry = JSON.parse(line); + if (entry?.kind === 0 && entry.v && typeof entry.v === "object") { + snapshot = entry.v as CopilotSession; + } + } catch { + // skip malformed line + } + } + if (!snapshot) return messages; + session = snapshot; } // --- Format 1: session.turns[] (older VS Code) @@ -206,7 +247,7 @@ export async function discoverCopilotSessions(options: { const sessions: CopilotSessionMeta[] = []; for (const root of roots) { - const glob = new Bun.Glob("*/chatSessions/*.json"); + const glob = new Bun.Glob("*/chatSessions/*.{json,jsonl}"); try { for await (const match of glob.scan({ cwd: root, absolute: false })) { const normalizedMatch = match.replaceAll("\\", "/"); @@ -216,7 +257,8 @@ export async function discoverCopilotSessions(options: { if (options.projectPath && workspacePath !== options.projectPath) continue; - const sessionId = `copilot-${basename(normalizedMatch, ".json")}`; + const ext = normalizedMatch.endsWith(".jsonl") ? ".jsonl" : ".json"; + const sessionId = `copilot-${basename(normalizedMatch, ext)}`; sessions.push({ sessionId, filePath, workspacePath }); } } catch { diff --git a/src/ingest/cursor.ts b/src/ingest/cursor.ts index 92a4c79..53d9a9b 100644 --- a/src/ingest/cursor.ts +++ b/src/ingest/cursor.ts @@ -1,14 +1,32 @@ /** * cursor.ts - Cursor IDE conversation parser * - * Reads conversation data from .cursor/ directories within projects - * and normalizes to QMD's addMessage() format. + * Reads conversation data from two sources: + * + * 1. Cursor app SQLite databases (primary): + * ~/Library/Application Support/Cursor/User/globalStorage/state.vscdb + * cursorDiskKV table: composerData: and bubbleId:: + * + * 2. Project .cursor/ JSON files (legacy / fallback): + * /.cursor/**\/*.json + * + * macOS: ~/Library/Application Support/Cursor/User/ + * Linux: ~/.config/Cursor/User/ + * Windows: %APPDATA%\Cursor\User\ + * + * Override root with CURSOR_STORAGE_DIR env var. */ -import { join } from "path"; -import { addMessage } from "../qmd"; +import { existsSync, readdirSync, readFileSync } from "fs"; +import { join, basename } from "path"; +import { homedir, platform } from "os"; +import { Database } from "bun:sqlite"; import type { ParsedMessage, IngestResult, IngestOptions } from "./index"; +// ============================================================================= +// Legacy JSON types (kept for .cursor/**/*.json backward compat) +// ============================================================================= + /** Shape of a Cursor conversation entry */ type CursorEntry = { role?: string; @@ -29,47 +47,323 @@ type CursorConversation = { }>; }; +// ============================================================================= +// SQLite / cursorDiskKV types +// ============================================================================= + +type ComposerData = { + _v?: number; + composerId: string; + richText?: string; + text?: string; + name?: string; + createdAt?: number; + /** Newer format: array of header objects referencing bubble IDs */ + fullConversationHeadersOnly?: Array<{ bubbleId: string; type: number }>; + /** Older format: inline messages */ + conversation?: Array<{ bubbleId: string; type: number; text?: string }>; +}; + +type BubbleData = { + type: number; // 1 = user, 2 = assistant + text?: string; + bubbleId?: string; + createdAt?: string; +}; + +// ============================================================================= +// Cursor user-dir root resolution +// ============================================================================= + /** - * Parse a Cursor conversation JSON file into normalized messages. + * Resolve Cursor's User directory roots across platforms. + * Returns all existing paths (checks both stable and non-stable editions). */ -export function parseCursorJson(content: string): ParsedMessage[] { - const messages: ParsedMessage[] = []; +export function resolveCursorUserRoots(): string[] { + const home = homedir(); + const candidates: string[] = []; - let data: CursorConversation | CursorConversation[]; + switch (platform()) { + case "darwin": + candidates.push( + join(home, "Library", "Application Support", "Cursor", "User"), + join(home, "Library", "Application Support", "Cursor - Insiders", "User") + ); + break; + case "linux": + candidates.push( + join(home, ".config", "Cursor", "User"), + join(home, ".config", "Cursor - Insiders", "User") + ); + break; + case "win32": + candidates.push( + join( + process.env.APPDATA || join(home, "AppData", "Roaming"), + "Cursor", + "User" + ), + join( + process.env.APPDATA || join(home, "AppData", "Roaming"), + "Cursor - Insiders", + "User" + ) + ); + break; + } + + // Allow override via env var + const envPath = Bun.env.CURSOR_STORAGE_DIR; + if (envPath) candidates.unshift(envPath); + + return candidates.filter(existsSync); +} + +// ============================================================================= +// Workspace mapping: composerId → project folder path +// ============================================================================= + +/** + * Read workspace.json to get the folder path for a workspaceStorage hash dir. + * Shape: { "folder": "file:///Users/..." } + */ +function readWorkspaceFolderPath(hashDir: string): string | null { + const wsJsonPath = join(hashDir, "workspace.json"); try { - data = JSON.parse(content); + const parsed = JSON.parse(readFileSync(wsJsonPath, "utf8")) as { + folder?: string; + workspace?: string; + }; + const raw = parsed.folder || parsed.workspace || null; + if (!raw) return null; + return decodeURIComponent(raw.replace(/^file:\/\//, "")); + } catch { + return null; + } +} + +/** + * Build a map of composerId → workspace folder path by scanning all + * workspaceStorage hash directories in the given Cursor User root. + * + * Each workspace state.vscdb has an ItemTable with key `composer.composerData` + * that contains JSON: { allComposers: [{ composerId, ... }] }. + */ +export function buildComposerWorkspaceMap(cursorUserRoot: string): Map { + const result = new Map(); + const wsStorageDir = join(cursorUserRoot, "workspaceStorage"); + if (!existsSync(wsStorageDir)) return result; + + let entries: string[]; + try { + entries = readdirSync(wsStorageDir); } catch { + return result; + } + + for (const hash of entries) { + const hashDir = join(wsStorageDir, hash); + const dbPath = join(hashDir, "state.vscdb"); + if (!existsSync(dbPath)) continue; + + const folderPath = readWorkspaceFolderPath(hashDir); + if (!folderPath) continue; + + let db: Database | null = null; + try { + db = new Database(dbPath, { readonly: true }); + const row = db.prepare( + `SELECT value FROM ItemTable WHERE key = 'composer.composerData' LIMIT 1` + ).get() as { value: string } | null; + + if (!row?.value) continue; + + const parsed = JSON.parse(row.value) as { + allComposers?: Array<{ composerId: string }>; + }; + + for (const c of parsed.allComposers ?? []) { + if (c.composerId) { + result.set(c.composerId, folderPath); + } + } + } catch { + // skip unreadable workspace DBs + } finally { + db?.close(); + } + } + + return result; +} + +// ============================================================================= +// SQLite-based session discovery +// ============================================================================= + +export type CursorSessionMeta = { + sessionId: string; + /** null means not associated with a known workspace */ + projectPath: string | null; + composerId: string; + createdAt: string; + title: string; +}; + +/** + * Parse composer JSON value into messages by resolving bubbles. + * Prefers inline `conversation` if non-empty, else falls back to + * fullConversationHeadersOnly + bubble lookup map. + */ +export function resolveComposerMessages( + composer: ComposerData, + bubbleMap: Map +): ParsedMessage[] { + const messages: ParsedMessage[] = []; + + const roleFromType = (type: number): "user" | "assistant" | null => + type === 1 ? "user" : type === 2 ? "assistant" : null; + + // Fallback timestamp: composer creation time (bubbles often carry none) + const composerTs = composer.createdAt + ? new Date(composer.createdAt).toISOString() + : undefined; + + // Prefer inline conversation array (older format) + if (composer.conversation && composer.conversation.length > 0) { + for (const entry of composer.conversation) { + const role = roleFromType(entry.type); + if (!role) continue; + const text = entry.text?.trim(); + if (!text) continue; + messages.push({ role, content: text, ...(composerTs ? { timestamp: composerTs } : {}) }); + } return messages; } - const conversations = Array.isArray(data) ? data : [data]; + // Newer format: headers-only + bubble lookup + if (composer.fullConversationHeadersOnly) { + for (const header of composer.fullConversationHeadersOnly) { + const role = roleFromType(header.type); + if (!role) continue; + const bubble = bubbleMap.get(header.bubbleId); + if (!bubble) continue; + const text = bubble.text?.trim(); + if (!text) continue; + const ts = bubble.createdAt || composerTs; + messages.push({ role, content: text, ...(ts ? { timestamp: ts } : {}) }); + } + } - for (const conv of conversations) { - const allMessages = [ - ...(conv.messages || []), - ...(conv.tabs?.flatMap((t) => t.messages || []) || []), - ]; + return messages; +} - for (const entry of allMessages) { - const role = entry.role || entry.type; - if (!role || (role !== "user" && role !== "assistant")) continue; +/** + * Discover Cursor sessions from a global storage SQLite database. + * + * @param globalDbPath Path to state.vscdb (globalStorage) + * @param composerWorkspaceMap pre-built composerId → folder path map + * @param projectPath Optional filter: only return composers in this workspace + */ +export function discoverCursorSqliteSessions( + globalDbPath: string, + composerWorkspaceMap: Map, + options: { projectPath?: string } = {} +): Array<{ meta: CursorSessionMeta; messages: ParsedMessage[] }> { + const results: Array<{ meta: CursorSessionMeta; messages: ParsedMessage[] }> = []; - const text = entry.content || entry.text; - if (!text?.trim()) continue; + let db: Database | null = null; + try { + db = new Database(globalDbPath, { readonly: true }); - messages.push({ - role, - content: text, - timestamp: entry.timestamp, + // Load all composer entries + const composerRows = db + .prepare( + `SELECT key, value FROM cursorDiskKV WHERE key LIKE 'composerData:%' AND value IS NOT NULL AND value != ''` + ) + .all() as { key: string; value: string }[]; + + // Load all bubble entries into a flat map: bubbleId -> BubbleData + // We batch-load only the bubbles we need after parsing composers + // For efficiency, load all into a single Map + const bubbleRows = db + .prepare( + `SELECT key, value FROM cursorDiskKV WHERE key LIKE 'bubbleId:%' AND value IS NOT NULL AND value != ''` + ) + .all() as { key: string; value: string }[]; + + // Map: composerId:bubbleId -> BubbleData + const bubbleMap = new Map(); + for (const row of bubbleRows) { + // key format: bubbleId:: + const parts = row.key.split(":"); + if (parts.length < 3) continue; + const bubbleId = parts[parts.length - 1]; + if (!bubbleId) continue; + try { + const data = JSON.parse(row.value) as BubbleData; + bubbleMap.set(bubbleId, data); + } catch { + // skip malformed bubble + } + } + + for (const row of composerRows) { + let composer: ComposerData; + try { + composer = JSON.parse(row.value) as ComposerData; + } catch { + continue; + } + + if (!composer.composerId) continue; + + const folderPath = composerWorkspaceMap.get(composer.composerId) ?? null; + + // Apply project filter if specified + if (options.projectPath && folderPath !== options.projectPath) continue; + + const messages = resolveComposerMessages(composer, bubbleMap); + if (messages.length === 0) continue; + + // Derive title from name, first user message, or richText + const firstUser = messages.find((m) => m.role === "user"); + const title = + composer.name?.trim() || + firstUser?.content.slice(0, 100).replace(/\n/g, " ") || + "Cursor Chat"; + + const createdAt = composer.createdAt + ? new Date(composer.createdAt).toISOString() + : new Date().toISOString(); + + results.push({ + meta: { + sessionId: `cursor-${composer.composerId}`, + projectPath: folderPath, + composerId: composer.composerId, + createdAt, + title, + }, + messages, }); } + } catch { + // unreadable DB — skip + } finally { + db?.close(); } - return messages; + return results; } +// ============================================================================= +// Legacy: JSON file discovery +// ============================================================================= + /** - * Discover Cursor conversation files in a project directory. + * Discover Cursor conversation files in a project's .cursor/ directory. + * Kept for backward compatibility with projects that have local JSON exports. */ export async function discoverCursorSessions( projectPath: string @@ -85,7 +379,9 @@ export async function discoverCursorSessions( const glob = new Bun.Glob("**/*.json"); for await (const match of glob.scan({ cwd: cursorDir, absolute: false })) { const normalizedMatch = match.replaceAll("\\", "/"); - const sessionId = `cursor-${normalizedMatch.replace(/\.json$/, "").replaceAll("/", "-")}`; + const sessionId = `cursor-${normalizedMatch + .replace(/\.json$/, "") + .replaceAll("/", "-")}`; sessions.push({ sessionId, filePath: join(cursorDir, normalizedMatch), @@ -100,14 +396,60 @@ export async function discoverCursorSessions( } /** - * Ingest Cursor sessions from a project directory. + * Parse a Cursor conversation JSON file into normalized messages. + * Used only for the legacy .cursor/**\/*.json path. + */ +export function parseCursorJson(content: string): ParsedMessage[] { + const messages: ParsedMessage[] = []; + + let data: CursorConversation | CursorConversation[]; + try { + data = JSON.parse(content); + } catch { + return messages; + } + + const conversations = Array.isArray(data) ? data : [data]; + + for (const conv of conversations) { + const allMessages = [ + ...(conv.messages || []), + ...(conv.tabs?.flatMap((t) => t.messages || []) || []), + ]; + + for (const entry of allMessages) { + const role = entry.role || entry.type; + if (!role || (role !== "user" && role !== "assistant")) continue; + + const text = entry.content || entry.text; + if (!text?.trim()) continue; + + messages.push({ + role, + content: text, + timestamp: entry.timestamp, + }); + } + } + + return messages; +} + +// ============================================================================= +// Ingestion entry point +// ============================================================================= + +/** + * Ingest Cursor sessions. + * + * Without projectPath: scans Cursor app's SQLite globalStorage for all composers. + * With projectPath: scans SQLite and also falls back to .cursor/*.json files. */ export async function ingestCursor( options: IngestOptions & { projectPath?: string } = {} ): Promise { const { db, onProgress, projectPath } = options; if (!db) throw new Error("Database required for ingestion"); - if (!projectPath) throw new Error("projectPath required for Cursor ingestion"); const { ingest } = await import("./index"); return ingest(db, "cursor", { projectPath, diff --git a/src/ingest/index.ts b/src/ingest/index.ts index edfe7db..7f73005 100644 --- a/src/ingest/index.ts +++ b/src/ingest/index.ts @@ -103,21 +103,24 @@ async function ingestParsedSessions( const correlationMap: ToolCorrelationMap = new Map(); if (useSessionTxn) db.exec("BEGIN IMMEDIATE"); try { - // Force mode: delete existing sidecar rows before re-processing + // Force mode: delete existing rows before re-processing — including + // memory_messages, otherwise re-ingest appends duplicates if (options.force && options.existingSessionIds.has(session.sessionId)) { deleteSidecarRows(db, session.sessionId); + db.prepare(`DELETE FROM memory_messages WHERE session_id = ?`).run(session.sessionId); } for (const msg of messagesToIngest) { const content = isStructuredMessage(msg) ? msg.plainText || "(structured content)" : msg.content; const messageOptions = isStructuredMessage(msg) ? { title: parsed.session.title, + timestamp: msg.timestamp, metadata: { ...msg.metadata, blocks: msg.blocks, }, } - : { title: parsed.session.title }; + : { title: parsed.session.title, timestamp: msg.timestamp }; const stored = await storeMessage(db, session.sessionId, msg.role, content, messageOptions); if (!stored.success) { @@ -267,30 +270,130 @@ export async function ingest( }); } case "cursor": { - if (!options.projectPath) { - return { - agent: "cursor", - sessionsFound: 0, - sessionsIngested: 0, - messagesIngested: 0, - skipped: 0, - errors: ["projectPath required for Cursor ingestion"], - }; - } - const { discoverCursorSessions } = await import("./cursor"); + const { + discoverCursorSessions, + discoverCursorSqliteSessions, + buildComposerWorkspaceMap, + resolveCursorUserRoots, + parseCursorJson, + } = await import("./cursor"); const { parseCursor } = await import("./parsers"); - const discovered = await discoverCursorSessions(options.projectPath); - const sessions = discovered.map((s) => ({ - sessionId: s.sessionId, - filePath: s.filePath, - projectDir: s.projectPath, - })); - return ingestParsedSessions(db, "cursor", sessions, parseCursor, { - existingSessionIds, - onProgress: options.onProgress, - explicitProjectId: options.projectId, - force: options.force, - }); + + // --- SQLite path: scan Cursor app globalStorage for all composers --- + const cursorUserRoots = resolveCursorUserRoots(); + const sqliteSessions: Array<{ + sessionId: string; + filePath: string; // unused sentinel for the parser + projectDir?: string; + _preResolved?: { messages: ParsedMessage[]; title: string; createdAt: string }; + }> = []; + + for (const userRoot of cursorUserRoots) { + const globalDbPath = `${userRoot}/globalStorage/state.vscdb`; + const composerMap = buildComposerWorkspaceMap(userRoot); + const discovered = discoverCursorSqliteSessions(globalDbPath, composerMap, { + projectPath: options.projectPath, + }); + for (const { meta, messages } of discovered) { + sqliteSessions.push({ + sessionId: meta.sessionId, + filePath: globalDbPath, + projectDir: meta.projectPath ?? undefined, + _preResolved: { messages, title: meta.title, createdAt: meta.createdAt }, + }); + } + } + + // --- Legacy path: .cursor/**/*.json (only if projectPath given) --- + const legacySessions: Array<{ + sessionId: string; + filePath: string; + projectDir?: string; + }> = []; + if (options.projectPath) { + const discovered = await discoverCursorSessions(options.projectPath); + // Avoid duplicating anything already covered by SQLite + const sqliteIds = new Set(sqliteSessions.map((s) => s.sessionId)); + for (const s of discovered) { + if (!sqliteIds.has(s.sessionId)) { + legacySessions.push({ + sessionId: s.sessionId, + filePath: s.filePath, + projectDir: s.projectPath, + }); + } + } + } + + // Ingest SQLite-discovered sessions with an identity parser (messages pre-resolved) + let sqliteResult: IngestResult = { + agent: "cursor", + sessionsFound: 0, + sessionsIngested: 0, + messagesIngested: 0, + skipped: 0, + errors: [], + }; + if (sqliteSessions.length > 0) { + sqliteResult = await ingestParsedSessions( + db, + "cursor", + sqliteSessions, + async (_filePath: string, sessionId: string) => { + // Find pre-resolved data for this sessionId + const entry = sqliteSessions.find((s) => s.sessionId === sessionId); + const pre = entry?._preResolved; + return { + session: { + id: sessionId, + title: pre?.title ?? "Cursor Chat", + created_at: pre?.createdAt ?? new Date().toISOString(), + }, + messages: pre?.messages ?? [], + }; + }, + { + existingSessionIds, + onProgress: options.onProgress, + explicitProjectId: options.projectId, + force: options.force, + } + ); + } + + // Ingest legacy JSON sessions + let legacyResult: IngestResult = { + agent: "cursor", + sessionsFound: 0, + sessionsIngested: 0, + messagesIngested: 0, + skipped: 0, + errors: [], + }; + if (legacySessions.length > 0) { + legacyResult = await ingestParsedSessions( + db, + "cursor", + legacySessions, + parseCursor, + { + existingSessionIds, + onProgress: options.onProgress, + explicitProjectId: options.projectId, + force: options.force, + } + ); + } + + // Merge results + return { + agent: "cursor", + sessionsFound: sqliteResult.sessionsFound + legacyResult.sessionsFound, + sessionsIngested: sqliteResult.sessionsIngested + legacyResult.sessionsIngested, + messagesIngested: sqliteResult.messagesIngested + legacyResult.messagesIngested, + skipped: sqliteResult.skipped + legacyResult.skipped, + errors: [...sqliteResult.errors, ...legacyResult.errors], + }; } case "cline": { const { discoverClineSessions } = await import("./cline"); diff --git a/src/ingest/store-gateway.ts b/src/ingest/store-gateway.ts index 8caa735..75fb890 100644 --- a/src/ingest/store-gateway.ts +++ b/src/ingest/store-gateway.ts @@ -9,7 +9,13 @@ import { upsertProject, upsertSessionCosts, upsertSessionMeta, + computeDensityScore, + updateDensityScore, + insertSessionQueries, + getSessionQueryCount, + writeSessionDocument, } from "../db"; +import { getQmdStore } from "../store"; import type { MessageBlock } from "./types"; export type StoreMessageResult = { @@ -26,7 +32,7 @@ export async function storeMessage( sessionId: string, role: string, content: string, - options?: { title?: string; metadata?: Record } + options?: { title?: string; metadata?: Record; timestamp?: string } ): Promise { try { const stored = await addMessage(db, sessionId, role, content, options); @@ -152,6 +158,40 @@ export function storeSession( .prepare(`SELECT 1 as yes FROM smriti_agents WHERE id = ?`) .get(agentId) as { yes: number } | null; upsertSessionMeta(db, sessionId, agentExists ? agentId : undefined, projectId || undefined); + + // Compute and persist density score after all sidecar rows are written + const { score } = computeDensityScore(db as any, sessionId); + updateDensityScore(db as any, sessionId, score); + + // Bulk backfills: skip LLM enrichment (query expansion) and collection sync — + // run `smriti enrich` / `smriti embed` afterwards instead + if (process.env.SMRITI_INGEST_NO_ENRICH === "1") return; + + // Write session markdown to QMD smriti-sessions collection (non-blocking, best-effort) + writeSessionDocument(db as any, sessionId, agentExists ? agentId : null, projectId).then(async () => { + try { + const store = getQmdStore(); + await store.update({ collections: ["smriti-sessions"] }); + } catch { /* collection not registered yet — ok */ } + }).catch(() => { /* sessions dir not configured — skip silently */ }); + + // Auto-enrich with query aliases (non-blocking, best-effort) + if (getSessionQueryCount(db as any, sessionId) === 0) { + const session = db.prepare(`SELECT title, summary FROM memory_sessions WHERE id = ?`).get(sessionId) as { title: string; summary: string | null } | null; + if (session?.title) { + const input = session.title + (session.summary ? ". " + session.summary : ""); + try { + const store = getQmdStore(); + // Fire-and-forget: don't await, never block ingest + store.internal.expandQuery(input).then((expanded) => { + const queryTexts = expanded.map(e => e.query).filter(Boolean); + insertSessionQueries(db as any, sessionId, queryTexts, "auto"); + }).catch(() => { /* LLM unavailable, skip silently */ }); + } catch { + // Store not initialized or LLM unavailable — skip silently + } + } + } } export function storeCosts( diff --git a/src/learn/consolidate.ts b/src/learn/consolidate.ts new file mode 100644 index 0000000..c57326f --- /dev/null +++ b/src/learn/consolidate.ts @@ -0,0 +1,612 @@ +/** + * learn/consolidate.ts - Continuous knowledge consolidation + * + * Progressive Summarization: cheap Stage-1 extraction runs broadly over dense + * sessions; expensive Stage-2 polish only runs once a unit proves it's reused + * (recalled repeatedly) or scored high relevance at extraction time. + * + * Two independent phases, run sequentially: + * - Segment: dense, not-yet-segmented sessions -> segmentSession() -> smriti_knowledge_units + * - Promote: knowledge units that cleared the reuse/relevance bar -> generateDocument() + * -> written to .smriti/knowledge/ + recorded in smriti_shares + * + * CLI-only, like `categorize`/`share` — never wired into the daemon (see + * src/daemon/index.ts's enrichOnIngest comment for why LLM work per-flush is unsafe). + */ + +import type { Database } from "bun:sqlite"; +import { mkdirSync } from "fs"; +import { join } from "path"; +import { SMRITI_DIR, AUTHOR } from "../config"; +import { hashContent } from "../qmd"; +import { + findUnsegmentedDenseSessions, + insertKnowledgeUnit, + findPromotableUnits, + promoteKnowledgeUnit, + findStaleSegmentedUnits, + findSupersededCanonicalUnits, + deleteKnowledgeUnit, + archiveKnowledgeUnit, +} from "../db"; +import { getSessionMessages } from "../team/share"; +import { segmentSession } from "../team/segment"; +import { generateDocument, generateFrontmatter } from "../team/document"; +import { isSessionWorthSharing } from "../team/formatter"; +import { callOllama } from "../team/ollama"; +import { ollamaChat, type OllamaTool } from "../ollama"; +import type { RawMessage } from "../team/formatter"; +import type { KnowledgeUnit } from "../team/types"; +import { + resolveEntity, + insertRelationship, + getRelationships, + findRelatedCandidates, + type RelationshipPredicate, +} from "./entities"; + +// ============================================================================= +// Types +// ============================================================================= + +export type ConsolidateOptions = { + minDensity?: number; + minRetrievals?: number; + minRelevance?: number; + minEntityReach?: number; + model?: string; + outputDir?: string; + author?: string; + sessionLimit?: number; + onProgress?: (msg: string) => void; + /** Also run the prune phase (dry-run by default — see pruneApply). */ + prune?: boolean; + /** Age threshold (days) for stale, never-promoted segmented units. Default 30. */ + pruneStaleDays?: number; + /** Actually delete/archive prune candidates. Without this, prune only reports candidates (dry-run). */ + pruneApply?: boolean; +}; + +export type PruneCandidate = { + id: string; + topic: string; + tier: "segmented" | "canonical"; + action: "delete" | "archive"; + reason: string; +}; + +export type ConsolidateResult = { + sessionsSegmented: number; + unitsStored: number; + unitsSkipped: number; + unitsPromoted: number; + /** Only set when options.prune is true. */ + unitsPruned?: number; + unitsArchived?: number; + pruneCandidates?: PruneCandidate[]; + errors: string[]; +}; + +// ============================================================================= +// Consolidation +// ============================================================================= + +export async function consolidateKnowledge( + db: Database, + options: ConsolidateOptions = {} +): Promise { + const author = options.author || AUTHOR; + const outputDir = options.outputDir || join(process.cwd(), SMRITI_DIR); + + const result: ConsolidateResult = { + sessionsSegmented: 0, + unitsStored: 0, + unitsSkipped: 0, + unitsPromoted: 0, + errors: [], + }; + + // =========================================================================== + // Segment phase: cheap Stage-1 extraction over dense, unsegmented sessions + // =========================================================================== + + const sessions = findUnsegmentedDenseSessions( + db, + options.minDensity ?? 0.5, + options.sessionLimit ?? 20 + ); + + for (const s of sessions) { + try { + const messages = getSessionMessages(db, s.session_id); + if (messages.length === 0) continue; + + const rawMessages: RawMessage[] = messages.map((m) => ({ + role: m.role, + content: m.content, + })); + + if (!isSessionWorthSharing(rawMessages)) continue; + + const segmentationResult = await segmentSession(db, s.session_id, rawMessages, { + model: options.model, + }); + result.sessionsSegmented++; + + for (const unit of segmentationResult.units) { + const contentHash = await hashContent( + JSON.stringify({ topic: unit.topic, category: unit.category, plainText: unit.plainText }) + ); + const inserted = insertKnowledgeUnit(db, unit, s.session_id, s.project_id, contentHash); + inserted ? result.unitsStored++ : result.unitsSkipped++; + + // Turn Stage 1's free-text entities into canonical "mentions" edges — + // pure post-processing of data already extracted, no extra LLM calls. + if (inserted) { + for (const rawEntity of unit.entities) { + const entityId = resolveEntity(db, rawEntity); + if (entityId) { + insertRelationship(db, "knowledge_unit", unit.id, "mentions", "entity", entityId, { + source: "extraction", + }); + } + } + } + } + } catch (err: any) { + result.errors.push(`segment ${s.session_id}: ${err.message}`); + } + } + + options.onProgress?.( + `segment phase: ${result.sessionsSegmented} sessions, ${result.unitsStored} units stored, ${result.unitsSkipped} skipped` + ); + + // =========================================================================== + // Promote phase: expensive Stage-2 polish for units that proved reuse + // =========================================================================== + + const knowledgeDir = join(outputDir, "knowledge"); + mkdirSync(knowledgeDir, { recursive: true }); + + const promotable = findPromotableUnits( + db, + options.minRetrievals ?? 3, + options.minRelevance ?? 8, + options.minEntityReach + ); + + for (const stored of promotable) { + try { + const unit: KnowledgeUnit = { + id: stored.id, + topic: stored.topic, + category: stored.category, + relevance: stored.relevance, + entities: stored.entities, + files: stored.files, + plainText: stored.plain_text, + lineRanges: stored.line_ranges, + }; + + // Bounded relationship inference: only runs if this unit shares a + // canonical entity with at least one other unit, and costs exactly one + // extra LLM call (same cost discipline as Stage 2) — persists what + // ollamaCheckConflicts previously only computed ephemerally. + const candidates = findRelatedCandidates(db, stored.id, 5); + if (candidates.length > 0) { + await inferRelationships(db, unit, candidates, options.model); + } + + const doc = await generateDocument(unit, stored.topic, { + model: options.model, + projectSmritiDir: outputDir, + author, + }); + + const categoryDir = join(knowledgeDir, doc.category.replaceAll("/", "-")); + mkdirSync(categoryDir, { recursive: true }); + const filePath = join(categoryDir, doc.filename); + + // Carry canonical entity ids + unit-to-unit edges into shared + // frontmatter. Unlike entity ids, these edges need no team-level + // canonicalization step — unit.id is already a portable UUID once + // shared (see src/team/document.ts's frontmatter `id` field), so + // syncTeamKnowledge can re-create them on a teammate's machine as-is. + const entityIds = getRelationships(db, { + subjectType: "knowledge_unit", + subjectId: stored.id, + predicate: "mentions", + objectType: "entity", + }).map((r) => r.object_id); + const outgoingEdges = getRelationships(db, { + subjectType: "knowledge_unit", + subjectId: stored.id, + }).filter((r) => r.predicate !== "mentions"); + + const fm = generateFrontmatter( + stored.session_id, + doc.unitId, + { + ...doc.frontmatter, + pipeline: "consolidated", + entity_ids: entityIds, + ...groupEdgesByPredicate(outgoingEdges), + }, + author, + stored.project_id || undefined + ); + await Bun.write(filePath, fm + "\n\n" + doc.markdown); + + const shareId = crypto.randomUUID().slice(0, 8); + const shareHash = await hashContent( + JSON.stringify({ content: doc.markdown, category: doc.category, entities: doc.frontmatter.entities }) + ); + + db.prepare( + `INSERT INTO smriti_shares (id, session_id, category_id, project_id, author, content_hash, unit_id, relevance_score, entities) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + shareId, + stored.session_id, + doc.category, + stored.project_id, + author, + shareHash, + doc.unitId, + stored.relevance, + JSON.stringify(stored.entities) + ); + + const relPath = `knowledge/${doc.category.replaceAll("/", "-")}/${doc.filename}`; + promoteKnowledgeUnit(db, stored.id, relPath, shareId); + result.unitsPromoted++; + } catch (err: any) { + result.errors.push(`promote ${stored.id}: ${err.message}`); + } + } + + options.onProgress?.(`promote phase: ${result.unitsPromoted} units promoted`); + + // =========================================================================== + // Prune phase: expire stale never-promoted units, archive superseded ones. + // Pure DB logic (age, retrieval_count, supersedes-edges are all already in + // SQLite by now) — no LLM call, dry-run by default. + // =========================================================================== + + if (options.prune) { + const pruneResult = await pruneKnowledge(db, { + outputDir, + pruneStaleDays: options.pruneStaleDays, + minRelevance: options.minRelevance, + dryRun: !options.pruneApply, + }); + result.unitsPruned = pruneResult.unitsPruned; + result.unitsArchived = pruneResult.unitsArchived; + result.pruneCandidates = pruneResult.pruneCandidates; + + options.onProgress?.( + pruneResult.pruneCandidates + ? `prune phase (dry-run): ${pruneResult.pruneCandidates.length} candidates — rerun with --yes to apply` + : `prune phase: ${pruneResult.unitsPruned} units deleted, ${pruneResult.unitsArchived} archived` + ); + } + + return result; +} + +// ============================================================================= +// Prune (expire stale segmented units, archive superseded canonical units) +// ============================================================================= + +export type PruneOptions = { + outputDir?: string; + pruneStaleDays?: number; + minRelevance?: number; + /** Report candidates without mutating the DB. Defaults to true — pass false to apply. */ + dryRun?: boolean; +}; + +export type PruneResult = { + unitsPruned: number; + unitsArchived: number; + pruneCandidates?: PruneCandidate[]; +}; + +export async function pruneKnowledge( + db: Database, + options: PruneOptions = {} +): Promise { + const dryRun = options.dryRun ?? true; + const outputDir = options.outputDir || join(process.cwd(), SMRITI_DIR); + const staleDays = options.pruneStaleDays ?? 30; + const minRelevance = options.minRelevance ?? 8; + + const stale = findStaleSegmentedUnits(db, staleDays, minRelevance); + const superseded = findSupersededCanonicalUnits(db); + + if (dryRun) { + const pruneCandidates: PruneCandidate[] = [ + ...stale.map((u) => ({ + id: u.id, + topic: u.topic, + tier: "segmented" as const, + action: "delete" as const, + reason: `stale segmented, 0 retrievals, relevance ${u.relevance} < ${minRelevance}`, + })), + ...superseded.map((u) => ({ + id: u.id, + topic: u.topic, + tier: "canonical" as const, + action: "archive" as const, + reason: `superseded by "${u.supersededByTopic}"`, + })), + ]; + return { unitsPruned: 0, unitsArchived: 0, pruneCandidates }; + } + + for (const u of stale) { + deleteKnowledgeUnit(db, u.id); + } + for (const u of superseded) { + archiveKnowledgeUnit(db, u.id, "superseded"); + await appendArchivedBanner(outputDir, u.canonical_doc_path, u.supersededByTopic); + } + + return { unitsPruned: stale.length, unitsArchived: superseded.length }; +} + +/** + * Prepend a short deprecation banner to an archived unit's canonical doc. + * The file itself is never deleted or moved — its path stays stable for + * anything already referencing it (team sync, a committed link) — only its + * content gains a notice pointing at the unit that superseded it. + */ +async function appendArchivedBanner( + outputDir: string, + docPath: string | null, + supersededByTopic: string +): Promise { + if (!docPath) return; + const filePath = join(outputDir, docPath); + const file = Bun.file(filePath); + if (!(await file.exists())) return; // doc already moved/removed outside Smriti — archive the DB row regardless + + const content = await file.text(); + const banner = `> **Archived** — superseded by "${supersededByTopic}".\n`; + const frontmatterMatch = content.match(/^---\n[\s\S]*?\n---\n/); + const updated = frontmatterMatch + ? content.slice(0, frontmatterMatch[0].length) + "\n" + banner + content.slice(frontmatterMatch[0].length) + : banner + "\n" + content; + + await Bun.write(filePath, updated); +} + +// ============================================================================= +// Relationship Inference (promote-time, LLM-gated) +// ============================================================================= + +const MAX_EXCERPT_CHARS = 800; + +export type RelationCandidate = { id: string; topic: string; category: string; plain_text: string }; +export type RelationGuess = { index: number; predicate: RelationshipPredicate | "none" }; + +function truncate(text: string, max: number): string { + return text.length > max ? text.slice(0, max) + "…" : text; +} + +function buildCandidateBlock(candidates: RelationCandidate[]): string { + return candidates + .map((c, i) => `[${i}] Topic: ${c.topic}\nCategory: ${c.category}\nContent: ${truncate(c.plain_text, MAX_EXCERPT_CHARS)}`) + .join("\n\n"); +} + +function buildComparisonPreamble( + unit: { topic: string; category: string; plainText: string }, + candidates: RelationCandidate[] +): string { + return `NEW UNIT +Topic: ${unit.topic} +Category: ${unit.category} +Content: ${truncate(unit.plainText, MAX_EXCERPT_CHARS)} + +CANDIDATES +${buildCandidateBlock(candidates)}`; +} + +// Brackets optional: models reliably get the index and predicate right but +// don't reliably reproduce "[i]" literally (observed: "RELATION 0: supersedes" +// instead of "RELATION [0]: supersedes") — a strict bracket requirement here +// silently drops otherwise-correct answers. +const RELATION_LINE = /RELATION\s*\[?(\d+)\]?:\s*(relatesTo|supersedes|contradicts|none)/gi; + +// Case-insensitive regex match -> canonical camelCase predicate (avoid a blind +// .toLowerCase() on the match, which would turn "relatesTo" into "relatesto"). +const PREDICATE_BY_LOWERCASE: Record = { + relatesto: "relatesTo", + supersedes: "supersedes", + contradicts: "contradicts", +}; + +/** + * Original approach: ask for free-text "RELATION [i]: predicate" lines and + * parse them with a regex. Kept only as the "before" baseline for the eval + * comparison against classifyRelationshipsToolCall — no longer wired into + * inferRelationships(). + */ +export async function classifyRelationshipsTextFormat( + unit: { topic: string; category: string; plainText: string }, + candidates: RelationCandidate[], + model?: string +): Promise { + const prompt = `You are comparing a NEW knowledge unit against CANDIDATE units that already mention at least one of the same topics/entities. + +${buildComparisonPreamble(unit, candidates)} + +For each candidate, decide the relationship of the NEW unit to it: +- relatesTo: related but neither replaces nor conflicts with the other +- supersedes: the NEW unit replaces/updates the candidate's guidance +- contradicts: the NEW unit conflicts with the candidate +- none: no meaningful relationship + +Respond with exactly one line per candidate, in this format: +RELATION [i]: relatesTo|supersedes|contradicts|none`; + + const response = await callOllama(prompt, { model }); + + const guesses: RelationGuess[] = []; + for (const match of response.matchAll(RELATION_LINE)) { + const index = Number(match[1]); + const raw = match[2]!.toLowerCase(); + const predicate = raw === "none" ? "none" : PREDICATE_BY_LOWERCASE[raw]; + if (!predicate || !candidates[index]) continue; + guesses.push({ index, predicate }); + } + + // A non-empty response that yields zero parsed lines almost always means + // the model drifted from the expected format, not that every candidate + // was genuinely unrelated — surface it instead of promoting in silence. + if (guesses.length === 0 && response.trim().length > 0) { + console.warn( + `classifyRelationshipsTextFormat: parsed 0 relation lines from a non-empty response — response may not match the expected format:`, + truncate(response, 300) + ); + } + + return guesses; +} + +const VALID_PREDICATES = new Set(["relatesTo", "supersedes", "contradicts", "none"]); + +const RECORD_RELATIONSHIPS_TOOL: OllamaTool = { + type: "function", + function: { + name: "record_relationships", + description: + "Record the relationship of the NEW knowledge unit to each CANDIDATE unit, one entry per candidate index.", + parameters: { + type: "object", + properties: { + relationships: { + type: "array", + items: { + type: "object", + properties: { + index: { + type: "integer", + description: "The candidate's [i] index as shown in the CANDIDATES list", + }, + predicate: { + type: "string", + enum: ["relatesTo", "supersedes", "contradicts", "none"], + description: + "relatesTo: related but neither replaces nor conflicts; supersedes: NEW unit replaces/updates the candidate; contradicts: NEW unit conflicts with the candidate; none: no meaningful relationship", + }, + }, + required: ["index", "predicate"], + }, + }, + }, + required: ["relationships"], + }, + }, +}; + +/** + * Ask the LLM to classify the NEW unit's relationship to each candidate via + * a native tool call instead of free-text lines — the model returns + * structured JSON directly, so there's no format to drift from and nothing + * to regex-parse. + */ +export async function classifyRelationshipsToolCall( + unit: { topic: string; category: string; plainText: string }, + candidates: RelationCandidate[], + model?: string +): Promise { + const prompt = `Compare the NEW knowledge unit against each CANDIDATE unit below, then call record_relationships with your assessment for every candidate index. + +${buildComparisonPreamble(unit, candidates)}`; + + const resp = await ollamaChat([{ role: "user", content: prompt }], { + model, + tools: [RECORD_RELATIONSHIPS_TOOL], + temperature: 0.1, + }); + + const call = resp.message.tool_calls?.find((c) => c.function.name === "record_relationships"); + if (!call) { + if (resp.message.content?.trim()) { + console.warn( + `classifyRelationshipsToolCall: model answered without calling record_relationships:`, + truncate(resp.message.content, 300) + ); + } + return []; + } + + const raw = call.function.arguments?.relationships; + if (!Array.isArray(raw)) return []; + + const guesses: RelationGuess[] = []; + for (const entry of raw) { + const index = Number((entry as any)?.index); + const predicate = (entry as any)?.predicate; + if (!Number.isInteger(index) || !candidates[index] || !VALID_PREDICATES.has(predicate)) continue; + guesses.push({ index, predicate }); + } + return guesses; +} + +/** + * Ask the LLM whether the unit being promoted relatesTo/supersedes/contradicts + * any of its entity-sharing candidates, and persist the answer as edges. + * Best-effort: a failure here (LLM down, unparseable response) is swallowed — + * it's enrichment on top of promotion, not a precondition for it. + */ +async function inferRelationships( + db: Database, + unit: KnowledgeUnit, + candidates: RelationCandidate[], + model?: string +): Promise { + try { + const guesses = await classifyRelationshipsToolCall(unit, candidates, model); + + for (const { index, predicate } of guesses) { + if (predicate === "none") continue; + const candidate = candidates[index]!; + + // Directional predicates shouldn't hold in both directions for the same + // pair. When two entity-sharing units are promoted in the same run, + // each independently asks "do I relate to/supersede the other" — if the + // candidate already asserted the reverse relation (e.g. its own + // promotion ran first in this batch), keep that one and skip the + // contradictory reverse edge rather than storing both. + const reverseAlreadyAsserted = getRelationships(db, { + subjectType: "knowledge_unit", + subjectId: candidate.id, + predicate, + objectType: "knowledge_unit", + objectId: unit.id, + }).length > 0; + if (reverseAlreadyAsserted) continue; + + insertRelationship(db, "knowledge_unit", unit.id, predicate, "knowledge_unit", candidate.id, { + source: "llm", + }); + } + } catch { + // Enrichment only — never block promotion on a failed/unparseable relation call. + } +} + +/** Group outgoing relationship edges by predicate into frontmatter-ready arrays of object unit ids. */ +function groupEdgesByPredicate( + edges: Array<{ predicate: string; object_id: string }> +): Record { + const grouped: Record = {}; + for (const edge of edges) { + (grouped[edge.predicate] ??= []).push(edge.object_id); + } + return grouped; +} diff --git a/src/learn/entities.ts b/src/learn/entities.ts new file mode 100644 index 0000000..470091d --- /dev/null +++ b/src/learn/entities.ts @@ -0,0 +1,232 @@ +/** + * learn/entities.ts - Canonical entity resolution + relationship triples + * + * RDF-inspired, not literal RDF: no URIs/Turtle/SPARQL. Subject/object are + * (type, id) pairs instead of global URIs, since Smriti is a local SQLite + * tool, not a web-facing linked-data endpoint. The useful ideas kept are: + * stable resource identity (so recurrence is detected across wording + * variance) and typed subject-predicate-object facts that survive team + * sharing (see src/team/config.ts's exportEntities/mergeEntities and + * src/team/document.ts's frontmatter for how these propagate org-wide). + * + * v1 entity resolution is exact-normalize only (case/whitespace/punctuation + * via slugify) — "JWT" and "jwt" merge, "JWT" and "JSON Web Token" do not. + * True synonym resolution needs semantic matching and is out of scope here. + */ + +import type { Database } from "bun:sqlite"; +import { slugify } from "../team/utils"; + +// ============================================================================= +// Types +// ============================================================================= + +export type EntityType = "technology" | "concept" | "file" | "pattern"; +export type RelationshipPredicate = "mentions" | "relatesTo" | "supersedes" | "contradicts"; +export type RelationshipSubjectType = "knowledge_unit" | "entity" | "session"; +export type RelationshipSource = "extraction" | "derived" | "llm"; + +export type StoredEntity = { + id: string; + label: string; + entity_type: EntityType; + aliases: string[]; + mention_count: number; + first_seen_at: string; +}; + +export type StoredRelationship = { + id: number; + subject_type: RelationshipSubjectType; + subject_id: string; + predicate: RelationshipPredicate; + object_type: RelationshipSubjectType; + object_id: string; + confidence: number; + source: RelationshipSource; + created_at: string; +}; + +type EntityRow = { + id: string; + label: string; + entity_type: string; + aliases: string; + mention_count: number; + first_seen_at: string; +}; + +function deserializeEntity(row: EntityRow): StoredEntity { + return { + ...row, + entity_type: row.entity_type as EntityType, + aliases: JSON.parse(row.aliases), + }; +} + +// ============================================================================= +// Entity Resolution +// ============================================================================= + +/** + * Resolve a raw, free-text entity label to a canonical entity id, creating + * the entity if it doesn't exist yet. Matching is exact-normalize (via + * slugify) — same case/whitespace variant collapses to one node; different + * wordings for the same concept do not (see module docstring). + */ +export function resolveEntity( + db: Database, + rawLabel: string, + entityType: EntityType = "concept" +): string | null { + const trimmed = rawLabel.trim(); + if (!trimmed) return null; + + const id = slugify(trimmed); + if (!id) return null; + + const existing = db + .prepare(`SELECT aliases FROM smriti_entities WHERE id = ?`) + .get(id) as { aliases: string } | null; + + if (existing) { + const aliases: string[] = JSON.parse(existing.aliases); + if (!aliases.includes(trimmed)) { + aliases.push(trimmed); + db.prepare( + `UPDATE smriti_entities SET aliases = ?, mention_count = mention_count + 1 WHERE id = ?` + ).run(JSON.stringify(aliases), id); + } else { + db.prepare(`UPDATE smriti_entities SET mention_count = mention_count + 1 WHERE id = ?`).run(id); + } + return id; + } + + db.prepare( + `INSERT INTO smriti_entities (id, label, entity_type, aliases, mention_count) + VALUES (?, ?, ?, ?, 1)` + ).run(id, trimmed, entityType, JSON.stringify([trimmed])); + return id; +} + +export function getEntity(db: Database, id: string): StoredEntity | null { + const row = db.prepare(`SELECT * FROM smriti_entities WHERE id = ?`).get(id) as EntityRow | null; + return row ? deserializeEntity(row) : null; +} + +/** Look up an entity by exact id, or by slugified/label match against a raw query string. */ +export function findEntity(db: Database, query: string): StoredEntity | null { + const bySlug = getEntity(db, slugify(query)); + if (bySlug) return bySlug; + + const row = db + .prepare(`SELECT * FROM smriti_entities WHERE LOWER(label) = LOWER(?)`) + .get(query.trim()) as EntityRow | null; + return row ? deserializeEntity(row) : null; +} + +/** Knowledge units that `mentions` a given canonical entity — the display side of `smriti graph `. */ +export function getUnitsForEntity( + db: Database, + entityId: string +): Array<{ id: string; topic: string; category: string; relevance: number; tier: string; retrieval_count: number }> { + return db + .prepare( + `SELECT ku.id, ku.topic, ku.category, ku.relevance, ku.tier, ku.retrieval_count + FROM smriti_relationships r + JOIN smriti_knowledge_units ku ON ku.id = r.subject_id + WHERE r.subject_type = 'knowledge_unit' AND r.object_type = 'entity' + AND r.predicate = 'mentions' AND r.object_id = ? AND ku.tier != 'archived' + ORDER BY ku.retrieval_count DESC, ku.relevance DESC` + ) + .all(entityId) as Array<{ + id: string; topic: string; category: string; relevance: number; tier: string; retrieval_count: number; + }>; +} + +export function listEntities(db: Database, limit?: number): StoredEntity[] { + const rows = ( + limit + ? db.prepare(`SELECT * FROM smriti_entities ORDER BY mention_count DESC LIMIT ?`).all(limit) + : db.prepare(`SELECT * FROM smriti_entities ORDER BY mention_count DESC`).all() + ) as EntityRow[]; + return rows.map(deserializeEntity); +} + +// ============================================================================= +// Relationship Triples +// ============================================================================= + +/** Insert a (subject, predicate, object) triple. Deduped via the table's UNIQUE constraint. */ +export function insertRelationship( + db: Database, + subjectType: RelationshipSubjectType, + subjectId: string, + predicate: RelationshipPredicate, + objectType: RelationshipSubjectType, + objectId: string, + options: { confidence?: number; source?: RelationshipSource } = {} +): void { + db.prepare( + `INSERT OR IGNORE INTO smriti_relationships + (subject_type, subject_id, predicate, object_type, object_id, confidence, source) + VALUES (?, ?, ?, ?, ?, ?, ?)` + ).run( + subjectType, + subjectId, + predicate, + objectType, + objectId, + options.confidence ?? 1.0, + options.source ?? "extraction" + ); +} + +export type TriplePattern = { + subjectType?: RelationshipSubjectType; + subjectId?: string; + predicate?: RelationshipPredicate; + objectType?: RelationshipSubjectType; + objectId?: string; +}; + +/** Single-pattern triple lookup — the basic-graph-pattern piece of SPARQL, simplified to one triple at a time. */ +export function getRelationships(db: Database, pattern: TriplePattern): StoredRelationship[] { + const conditions: string[] = []; + const params: any[] = []; + + if (pattern.subjectType) { conditions.push("subject_type = ?"); params.push(pattern.subjectType); } + if (pattern.subjectId) { conditions.push("subject_id = ?"); params.push(pattern.subjectId); } + if (pattern.predicate) { conditions.push("predicate = ?"); params.push(pattern.predicate); } + if (pattern.objectType) { conditions.push("object_type = ?"); params.push(pattern.objectType); } + if (pattern.objectId) { conditions.push("object_id = ?"); params.push(pattern.objectId); } + + const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : ""; + return db.prepare(`SELECT * FROM smriti_relationships ${where}`).all(...params) as StoredRelationship[]; +} + +/** + * Find other knowledge units that `mentions` at least one of the same + * canonical entities as `unitId` — the candidate set for promote-time + * LLM relationship inference (bounded, so that call stays cheap). + */ +export function findRelatedCandidates( + db: Database, + unitId: string, + limit: number = 5 +): Array<{ id: string; topic: string; category: string; plain_text: string }> { + return db + .prepare( + `SELECT DISTINCT ku.id, ku.topic, ku.category, ku.plain_text + FROM smriti_relationships r1 + JOIN smriti_relationships r2 + ON r1.object_id = r2.object_id + AND r2.object_type = 'entity' AND r2.predicate = 'mentions' + JOIN smriti_knowledge_units ku ON ku.id = r2.subject_id + WHERE r1.subject_type = 'knowledge_unit' AND r1.subject_id = ? + AND r1.object_type = 'entity' AND r1.predicate = 'mentions' + AND r2.subject_id != r1.subject_id + LIMIT ?` + ) + .all(unitId, limit) as Array<{ id: string; topic: string; category: string; plain_text: string }>; +} diff --git a/src/memory.ts b/src/memory.ts index 47be654..f5045c1 100644 --- a/src/memory.ts +++ b/src/memory.ts @@ -18,17 +18,20 @@ import type { Database } from "../qmd/src/db"; import { hashContent, chunkDocumentByTokens, - insertEmbedding, reciprocalRankFusion, - type RankedResult, -} from "../qmd/src/store.js"; -import { - getDefaultLlamaCpp, formatQueryForEmbedding, formatDocForEmbedding, -} from "../qmd/src/llm.js"; + type RankedResult, +} from "../qmd/src/store.js"; +import { getQmdStore } from "./store"; import { ollamaSummarize, ollamaRecall as ollamaRecallSynthesize } from "./ollama"; +// Returns the LLM instance from the SDK store (set during initSmriti). +// Throws if called before initSmriti() — only vector search + embed paths use this. +function getMemoryLlm() { + return getQmdStore().internal.llm!; +} + // ============================================================================= // Types // ============================================================================= @@ -254,6 +257,44 @@ export function clearAllSessions(db: Database, hard: boolean = false): number { } } +/** + * Remove content_vectors/vectors_vec rows whose hash is no longer referenced + * by any memory message or active QMD document. Scoped deletion — unlike + * QMD's own cleanupOrphanedVectors (which only checks `documents` and would + * wipe every memory-message embedding, since messages aren't rows in + * `documents`). Called after a hard session delete; a no-op (returns 0) when + * the vector/document tables aren't present (e.g. sqlite-vec unavailable, or + * a minimal test schema that skipped createStore()). + */ +export function cleanupOrphanedMemoryVectors(db: Database): number { + try { + db.prepare(`SELECT 1 FROM vectors_vec LIMIT 0`).get(); + db.prepare(`SELECT 1 FROM documents LIMIT 0`).get(); + db.prepare(`SELECT 1 FROM content_vectors LIMIT 0`).get(); + } catch { + return 0; + } + + const orphanWhere = ` + NOT EXISTS (SELECT 1 FROM memory_messages m WHERE m.hash = content_vectors.hash) + AND NOT EXISTS (SELECT 1 FROM documents d WHERE d.hash = content_vectors.hash AND d.active = 1) + `; + + const { c } = db + .prepare(`SELECT COUNT(*) as c FROM content_vectors WHERE ${orphanWhere}`) + .get() as { c: number }; + if (c === 0) return 0; + + db.exec(` + DELETE FROM vectors_vec WHERE hash_seq IN ( + SELECT content_vectors.hash || '_' || content_vectors.seq FROM content_vectors WHERE ${orphanWhere} + ) + `); + db.exec(`DELETE FROM content_vectors WHERE ${orphanWhere}`); + + return c; +} + // ============================================================================= // Message CRUD // ============================================================================= @@ -266,9 +307,11 @@ export async function addMessage( sessionId: string, role: string, content: string, - options: { title?: string; metadata?: Record } = {} + options: { title?: string; metadata?: Record; timestamp?: string } = {} ): Promise { const now = new Date().toISOString(); + // Backfilled ingests pass the original message timestamp; live writes default to now + const created = options.timestamp || now; const hash = await hashContent(content); // Preserve "new" behavior, which generates an ID. @@ -281,7 +324,7 @@ export async function addMessage( db.prepare( `INSERT OR IGNORE INTO memory_sessions (id, title, created_at, updated_at, active) VALUES (?, ?, ?, ?, 1)` - ).run(resolvedSessionId, options.title || "", now, now); + ).run(resolvedSessionId, options.title || "", created, created); // If title is provided later, fill it only when current title is empty. if (options.title) { @@ -301,11 +344,11 @@ export async function addMessage( `INSERT INTO memory_messages (session_id, role, content, hash, created_at, metadata) VALUES (?, ?, ?, ?, ?, ?)` ) - .run(resolvedSessionId, role, content, hash, now, metadataStr); + .run(resolvedSessionId, role, content, hash, created, metadataStr); // Update session timestamp db.prepare(`UPDATE memory_sessions SET updated_at = ? WHERE id = ?`).run( - now, + created, resolvedSessionId ); @@ -315,7 +358,7 @@ export async function addMessage( role, content, hash, - created_at: now, + created_at: created, metadata: options.metadata || null, }; } @@ -442,7 +485,7 @@ export async function searchMemoryVec( if (!tableExists) return []; // Get query embedding - const llm = getDefaultLlamaCpp(); + const llm = getMemoryLlm(); const formattedQuery = formatQueryForEmbedding(query); const result = await llm.embed(formattedQuery, { isQuery: true }); if (!result) return []; @@ -549,7 +592,7 @@ export async function embedMemoryMessages( if (unembedded.length === 0) return 0; - const llm = getDefaultLlamaCpp(); + const llm = getMemoryLlm(); let embedded = 0; for (const msg of unembedded) { @@ -579,8 +622,7 @@ export async function embedMemoryMessages( const now = new Date().toISOString(); // Insert first chunk embedding - insertEmbedding( - db, + getQmdStore().internal.insertEmbedding( msg.hash, 0, chunks[0]!.pos, @@ -595,8 +637,7 @@ export async function embedMemoryMessages( const text = formatDocForEmbedding(chunk.text); const embedResult = await llm.embed(text); if (embedResult) { - insertEmbedding( - db, + getQmdStore().internal.insertEmbedding( msg.hash, i, chunk.pos, @@ -681,6 +722,7 @@ export async function summarizeRecentSessions( /** * Recall relevant memories for a query. * Combines FTS + vector search using RRF, deduplicates by session, + * optionally expands query + reranks (skipped when fast=true), * and optionally synthesizes via Ollama. */ export async function recallMemories( @@ -691,20 +733,27 @@ export async function recallMemories( synthesize?: boolean; model?: string; maxTokens?: number; + fast?: boolean; + intent?: string; } = {} ): Promise<{ results: MemorySearchResult[]; synthesis?: string }> { const startedAt = performance.now(); const shouldTraceRecall = process.env.SMRITI_BENCH_TRACE === "1"; const limit = options.limit ?? 10; + const fast = options.fast ?? false; + const intent = options.intent; - // Run FTS and vector search + // Candidate fetch size — fetch more when reranking to feed the reranker + const candidateLimit = fast ? limit : Math.max(limit * 4, 40); + + // Run FTS and vector search for the original query const ftsStartedAt = performance.now(); - const ftsResults = searchMemoryFTS(db, query, limit); + const ftsResults = searchMemoryFTS(db, query, candidateLimit); const ftsMs = performance.now() - ftsStartedAt; let vecResults: MemorySearchResult[] = []; const vecStartedAt = performance.now(); try { - vecResults = await searchMemoryVec(db, query, limit); + vecResults = await searchMemoryVec(db, query, candidateLimit); } catch { // Vector search may fail if no embeddings exist } @@ -720,12 +769,38 @@ export async function recallMemories( score: r.score, })); - // Fuse results with RRF + // Build ranked lists — start with original query results + const rankedLists: RankedResult[][] = [toRanked(ftsResults), toRanked(vecResults)]; + const rankWeights: number[] = [1.0, 1.0]; + + // Quality mode: expand query variants and fold in their results + if (!fast) { + try { + const store = getQmdStore(); + const expanded = await store.internal.expandQuery(query); + for (const variant of expanded) { + // lex variants are best suited for FTS; vec/hyde for vector search + const variantFts = searchMemoryFTS(db, variant.query, candidateLimit); + rankedLists.push(toRanked(variantFts)); + rankWeights.push(0.7); + if (variant.type !== "lex") { + try { + const variantVec = await searchMemoryVec(db, variant.query, candidateLimit); + rankedLists.push(toRanked(variantVec)); + rankWeights.push(0.7); + } catch { + // skip if no embeddings + } + } + } + } catch { + // LLM unavailable — fall through with original results only + } + } + + // Fuse all ranked lists with RRF const fuseStartedAt = performance.now(); - const fused = reciprocalRankFusion( - [toRanked(ftsResults), toRanked(vecResults)], - [1.0, 1.0] - ); + const fused = reciprocalRankFusion(rankedLists, rankWeights); const fuseMs = performance.now() - fuseStartedAt; // Deduplicate by session, keeping best score per session @@ -767,6 +842,57 @@ export async function recallMemories( } const dedupeMs = performance.now() - dedupeStartedAt; + // Quality mode: rerank the deduped candidates before density blending + if (!fast && dedupedResults.length > 1) { + try { + const store = getQmdStore(); + const docs = dedupedResults.map((r) => ({ + file: `${r.session_id}:${r.message_id}`, + text: r.content, + })); + const reranked = await store.internal.rerank(query, docs, undefined, intent); + const scoreMap = new Map(reranked.map((r) => [r.file, r.score])); + for (const r of dedupedResults) { + const key = `${r.session_id}:${r.message_id}`; + const rerankerScore = scoreMap.get(key); + if (rerankerScore !== undefined) { + // Blend: 60% reranker + 40% RRF to stay anchored to retrieval signal + r.score = rerankerScore * 0.6 + r.score * 0.4; + } + } + dedupedResults.sort((a, b) => b.score - a.score); + } catch { + // Reranker unavailable — keep RRF order + } + } + + // Blend density scores into recall scores — dense sessions rank higher. + // smriti_session_meta is a Smriti-layer table, not a QMD core one — this + // file is meant to stay usable against a bare QMD store (e.g. + // scripts/bench-qmd.ts), so a missing table degrades gracefully instead + // of throwing, same as the vector-search fallback above. + if (dedupedResults.length > 0) { + try { + const sessionIds = dedupedResults.map((r) => r.session_id); + const placeholders = sessionIds.map(() => "?").join(","); + const densityRows = (db as any) + .prepare( + `SELECT session_id, COALESCE(density_score, 0) as density_score + FROM smriti_session_meta WHERE session_id IN (${placeholders})` + ) + .all(...sessionIds) as { session_id: string; density_score: number }[]; + const densityMap = new Map(densityRows.map((r) => [r.session_id, r.density_score])); + + for (const r of dedupedResults) { + const ds = densityMap.get(r.session_id) ?? 0; + r.score = r.score * 0.8 + ds * 0.2; + } + dedupedResults.sort((a, b) => b.score - a.score); + } catch { + // smriti_session_meta doesn't exist (bare QMD store) — skip blending. + } + } + const results = dedupedResults.slice(0, limit); // Optionally synthesize via Ollama diff --git a/src/ollama.ts b/src/ollama.ts index bc3d082..f316209 100644 --- a/src/ollama.ts +++ b/src/ollama.ts @@ -6,29 +6,40 @@ * * Config via env: * OLLAMA_HOST - Ollama server URL (default: http://127.0.0.1:11434) - * QMD_MEMORY_MODEL - Model for summarization/synthesis (default: qwen3:8b-tuned) + * QMD_MEMORY_MODEL - Model for summarization/synthesis (required, no default) */ -// ============================================================================= -// Configuration -// ============================================================================= - -const OLLAMA_HOST = Bun.env.OLLAMA_HOST || "http://127.0.0.1:11434"; -const DEFAULT_MEMORY_MODEL = Bun.env.QMD_MEMORY_MODEL || "qwen3:8b-tuned"; +import { OLLAMA_HOST, requireOllamaModel } from "./config"; // ============================================================================= // Types // ============================================================================= +export type OllamaToolCall = { + id?: string; + function: { name: string; arguments: Record }; +}; + export type OllamaChatMessage = { - role: "system" | "user" | "assistant"; + role: "system" | "user" | "assistant" | "tool"; content: string; + tool_calls?: OllamaToolCall[]; +}; + +export type OllamaTool = { + type: "function"; + function: { + name: string; + description: string; + parameters: Record; + }; }; export type OllamaChatOptions = { model?: string; temperature?: number; maxTokens?: number; + tools?: OllamaTool[]; }; export type OllamaChatResponse = { @@ -51,7 +62,7 @@ export async function ollamaChat( messages: OllamaChatMessage[], options: OllamaChatOptions = {} ): Promise { - const model = options.model || DEFAULT_MEMORY_MODEL; + const model = requireOllamaModel(options.model); const resp = await fetch(`${OLLAMA_HOST}/api/chat`, { method: "POST", headers: { "Content-Type": "application/json" }, @@ -59,6 +70,7 @@ export async function ollamaChat( model, messages, stream: false, + ...(options.tools && { tools: options.tools }), options: { ...(options.temperature !== undefined && { temperature: options.temperature }), ...(options.maxTokens !== undefined && { num_predict: options.maxTokens }), @@ -139,6 +151,129 @@ export async function ollamaRecall( return resp.message.content.trim(); } +/** + * Answer a natural language question grounded in retrieved session memories. + * Returns the answer only — caller appends citations. + */ +export async function ollamaAsk( + question: string, + sources: string, + options: OllamaChatOptions = {} +): Promise { + const messages: OllamaChatMessage[] = [ + { + role: "system", + content: + "You are an expert assistant with access to an engineer's work history. " + + "Answer the question directly and precisely using only the provided sources. " + + "If sources are insufficient, say so explicitly. " + + "Cite sources by [N] where N is the source number. " + + "Be concise — answer in 2-4 sentences unless depth is needed. " + + "Output only the answer, no preamble.", + }, + { + role: "user", + content: `Question: ${question}\n\nSources:\n${sources}`, + }, + ]; + + const resp = await ollamaChat(messages, { + ...options, + temperature: options.temperature ?? 0.2, + maxTokens: options.maxTokens ?? 512, + }); + + return resp.message.content.trim(); +} + +/** + * Synthesize how thinking on a topic evolved across a chronological session timeline. + * Returns a narrative string showing key shifts, decisions, and reversals. + */ +export async function ollamaDrift( + topic: string, + timeline: string, + options: OllamaChatOptions = {} +): Promise { + const messages: OllamaChatMessage[] = [ + { + role: "system", + content: + "You are an engineering historian. Given a chronological timeline of sessions about a topic, " + + "describe how the team's thinking evolved. Focus on: decisions made, approaches tried, " + + "reversals, refinements, and the current state. Use a compact timeline format: " + + "one sentence per significant change. Highlight turning points. Output only the narrative.", + }, + { + role: "user", + content: `Topic: ${topic}\n\nTimeline:\n${timeline}`, + }, + ]; + + const resp = await ollamaChat(messages, { + ...options, + temperature: options.temperature ?? 0.3, + maxTokens: options.maxTokens ?? 768, + }); + + return resp.message.content.trim(); +} + +export type ConflictResult = { + pair: [number, number]; + description: string; +}; + +/** + * Detect contradictory pairs among recall results. + * Returns a list of conflicting index pairs with descriptions. + * Uses one Ollama call for all pairs (batch approach). + */ +export async function ollamaCheckConflicts( + topic: string, + passages: { n: number; title: string; content: string }[], + options: OllamaChatOptions = {} +): Promise { + const formatted = passages + .map(p => `[${p.n}] ${p.title}\n${p.content.slice(0, 300)}`) + .join("\n\n---\n\n"); + + const messages: OllamaChatMessage[] = [ + { + role: "system", + content: + "You detect contradictions between engineering decision records. " + + "Given numbered passages about the same topic, identify pairs that express conflicting approaches. " + + "Format each conflict as: CONFLICT [i] vs [j]: one-sentence description\n" + + "If no conflicts exist, output only: NO_CONFLICTS", + }, + { + role: "user", + content: `Topic: ${topic}\n\nPassages:\n${formatted}`, + }, + ]; + + const resp = await ollamaChat(messages, { + ...options, + temperature: 0.1, + maxTokens: options.maxTokens ?? 256, + }); + + const text = resp.message.content.trim(); + if (text.startsWith("NO_CONFLICTS")) return []; + + const results: ConflictResult[] = []; + const conflictRe = /CONFLICT\s+\[(\d+)\]\s+vs\s+\[(\d+)\]:\s*(.+)/gi; + let match; + while ((match = conflictRe.exec(text)) !== null) { + results.push({ + pair: [parseInt(match[1]!), parseInt(match[2]!)], + description: match[3]!.trim(), + }); + } + return results; +} + /** * Check if Ollama is running and accessible. * Pings the /api/tags endpoint. @@ -165,5 +300,3 @@ export async function ollamaHealthCheck(): Promise<{ }; } } - -export { DEFAULT_MEMORY_MODEL, OLLAMA_HOST }; diff --git a/src/qmd.ts b/src/qmd.ts index d474c90..6f4f6c8 100644 --- a/src/qmd.ts +++ b/src/qmd.ts @@ -17,6 +17,9 @@ export { importTranscript, initializeMemoryTables, createSession, + deleteSession, + clearAllSessions, + cleanupOrphanedMemoryVectors, } from "./memory"; export { hashContent } from "../qmd/src/store"; diff --git a/src/search/index.ts b/src/search/index.ts index 9cc30b0..2021fa3 100644 --- a/src/search/index.ts +++ b/src/search/index.ts @@ -134,7 +134,83 @@ export function searchFiltered( LIMIT ? `; - return db.prepare(sql).all(...params) as SearchResult[]; + const ftsRows = db.prepare(sql).all(...params) as SearchResult[]; + + // Also search query aliases (from smriti enrich --queries) and merge in sessions + // not already surfaced by FTS. + const labelRows = searchByQueryAliases(db, query, filters, limit); + const seenSessions = new Set(ftsRows.map(r => r.session_id)); + const novelFromLabels = labelRows.filter(r => !seenSessions.has(r.session_id)); + + return [...ftsRows, ...novelFromLabels].slice(0, limit); +} + +// ============================================================================= +// Query Alias Search (#60) +// ============================================================================= + +/** + * Find sessions via smriti_queries_fts (enriched aliases) and return their + * representative top message as SearchResults with source='query_alias'. + */ +function searchByQueryAliases( + db: Database, + query: string, + filters: SearchFilters, + limit: number +): SearchResult[] { + // Check table exists (enrichment is optional) + const tableExists = (db as any) + .prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='smriti_queries_fts'`) + .get(); + if (!tableExists) return []; + + const aliasConditions: string[] = ["smriti_queries_fts MATCH ?", "ms.active = 1"]; + const aliasParams: any[] = [query]; + + if (filters.project) { + aliasConditions.push("sm.project_id = ?"); + aliasParams.push(filters.project); + } + if (filters.agent) { + aliasConditions.push("sm.agent_id = ?"); + aliasParams.push(filters.agent); + } + if (filters.category) { + aliasConditions.push( + `EXISTS (SELECT 1 FROM smriti_session_tags st WHERE st.session_id = sq.session_id + AND (st.category_id = ? OR st.category_id LIKE ? || '/%'))` + ); + aliasParams.push(filters.category, filters.category); + } + aliasParams.push(limit); + + const sql = ` + SELECT DISTINCT + mm.session_id, + ms.title AS session_title, + mm.id AS message_id, + mm.role, + mm.content, + 0.5 AS score, + 'query_alias' AS source, + sm.project_id AS project, + sm.agent_id AS agent + FROM smriti_queries_fts + JOIN smriti_session_queries sq ON sq.rowid = smriti_queries_fts.rowid + JOIN memory_sessions ms ON ms.id = sq.session_id + JOIN memory_messages mm ON mm.session_id = sq.session_id + AND mm.id = (SELECT MIN(id) FROM memory_messages WHERE session_id = sq.session_id) + LEFT JOIN smriti_session_meta sm ON sm.session_id = sq.session_id + WHERE ${aliasConditions.join(" AND ")} + LIMIT ? + `; + + try { + return (db as any).prepare(sql).all(...aliasParams) as SearchResult[]; + } catch { + return []; + } } // ============================================================================= diff --git a/src/search/recall.ts b/src/search/recall.ts index a61b03d..f94fe7b 100644 --- a/src/search/recall.ts +++ b/src/search/recall.ts @@ -8,6 +8,8 @@ import type { Database } from "bun:sqlite"; import { DEFAULT_RECALL_LIMIT, OLLAMA_HOST, OLLAMA_MODEL } from "../config"; import { recallMemories, ollamaRecall } from "../qmd"; import { searchFiltered, type SearchFilters, type SearchResult } from "./index"; +import { getQmdStore } from "../store"; +import { incrementRetrievalCount } from "../db"; // ============================================================================= // Types @@ -17,6 +19,8 @@ export type RecallOptions = SearchFilters & { synthesize?: boolean; model?: string; maxTokens?: number; + fast?: boolean; + wide?: boolean; }; export type RecallResult = { @@ -24,6 +28,24 @@ export type RecallResult = { synthesis?: string; }; +// ============================================================================= +// Retrieval Tracking +// ============================================================================= + +/** + * Best-effort bump of retrieval_count for any consolidated knowledge units + * belonging to the recalled sessions. Never lets a tracking failure break recall. + */ +function trackRetrieval(db: Database, results: SearchResult[]): void { + try { + for (const sessionId of new Set(results.map((r) => r.session_id).filter(Boolean))) { + incrementRetrievalCount(db, sessionId); + } + } catch { + // Never let this break recall. + } +} + // ============================================================================= // Filtered Recall // ============================================================================= @@ -37,18 +59,38 @@ export async function recall( query: string, options: RecallOptions = {} ): Promise { - const hasFilters = options.category || options.project || options.agent + // --wide bypasses the project filter: search all projects, rerank with project as intent + const effectiveProject = (options.wide && options.project) ? undefined : options.project; + const rerankIntent = (options.wide && options.project) + ? `relevant to ${options.project} project context` + : undefined; + + const hasFilters = options.category || effectiveProject || options.agent || options.includeThinking || options.includeArtifacts === false || options.includeAttachments === false || options.includeVoiceNotes === false; if (!hasFilters) { - // Use QMD's native recall for unfiltered queries + // When smriti-sessions QMD collection has documents, use store.search() for full hybrid pipeline + const storeResults = await tryQmdSessionSearch(query, options.limit || DEFAULT_RECALL_LIMIT, rerankIntent, options.fast); + if (storeResults) { + let synthesis: string | undefined; + if (options.synthesize && storeResults.length > 0) { + synthesis = await synthesizeResults(query, storeResults, options); + } + trackRetrieval(db, storeResults); + return { results: storeResults, synthesis }; + } + + // Fallback: QMD's memory recall (recallMemories) const qmdResult = await recallMemories(db, query, { limit: options.limit || DEFAULT_RECALL_LIMIT, synthesize: options.synthesize, model: options.model, maxTokens: options.maxTokens, + fast: options.fast, + intent: rerankIntent, }); + trackRetrieval(db, qmdResult.results); return { results: qmdResult.results, synthesis: qmdResult.synthesis, @@ -58,7 +100,7 @@ export async function recall( // Filtered recall const results = searchFiltered(db, query, { category: options.category, - project: options.project, + project: effectiveProject, agent: options.agent, limit: options.limit || DEFAULT_RECALL_LIMIT, includeThinking: options.includeThinking, @@ -81,9 +123,53 @@ export async function recall( synthesis = await synthesizeResults(query, deduped, options); } + trackRetrieval(db, deduped); return { results: deduped, synthesis }; } +/** + * Try QMD store.search() on the smriti-sessions collection. + * Returns null if the collection doesn't exist or has no documents. + * Maps HybridQueryResult[] to SearchResult[] so callers are unchanged. + */ +async function tryQmdSessionSearch( + query: string, + limit: number, + intent?: string, + fast?: boolean +): Promise { + try { + const store = getQmdStore(); + const collections = await store.listCollections(); + const sessionsCol = collections.find(c => c.name === "smriti-sessions"); + if (!sessionsCol || sessionsCol.doc_count === 0) return null; + + const results = await store.search({ + query, + collections: ["smriti-sessions"], + limit, + intent, + rerank: !fast, + }); + + return results.map(r => { + // Extract session_id from the file path: smriti-sessions/.md + const filename = r.file.split("/").pop()?.replace(/\.md$/, "") ?? r.file; + return { + session_id: filename, + session_title: r.title, + message_id: 0, + role: "session", + content: r.bestChunk || r.body.slice(0, 500), + score: r.score, + source: "qmd", + } as SearchResult; + }); + } catch { + return null; + } +} + /** * Synthesize search results into a coherent summary using Ollama. */ diff --git a/src/store.ts b/src/store.ts new file mode 100644 index 0000000..6c58c84 --- /dev/null +++ b/src/store.ts @@ -0,0 +1,32 @@ +/** + * store.ts - QMD store singleton + * + * Holds the SDK-created QMDStore so both db.ts and memory.ts can access it + * without creating a circular dependency chain. + * No imports from db/memory/qmd — those all import from here, not vice versa. + */ + +import type { QMDStore } from "../qmd/src/index"; + +let _store: QMDStore | null = null; + +export function setQmdStore(s: QMDStore): void { + _store = s; +} + +export function getQmdStore(): QMDStore { + if (!_store) throw new Error("QMD store not initialized — call initSmriti() first"); + return _store; +} + +export async function closeQmdStore(): Promise { + if (_store) { + // Dispose the LlamaCpp/Llama backend explicitly instead of leaving it + // to the 5-min inactivity timer — otherwise a fresh Store created on + // the next initSmriti() call (e.g. the daemon's per-flush open) can + // overlap with the still-loaded previous instance. + try { await _store.internal.llm?.dispose(); } catch { /* best-effort */ } + _store.internal.close(); + _store = null; + } +} diff --git a/src/team/config.ts b/src/team/config.ts new file mode 100644 index 0000000..13c0efe --- /dev/null +++ b/src/team/config.ts @@ -0,0 +1,195 @@ +/** + * team/config.ts - .smriti/config.json schema and read/write/merge utilities + */ + +import type { Database } from "bun:sqlite"; +import { join } from "path"; +import { ALL_CATEGORY_IDS } from "../categorize/schema"; + +// ============================================================================= +// Types +// ============================================================================= + +export type CustomCategoryDef = { + id: string; + name: string; + parent?: string; + description?: string; +}; + +export type CustomEntityDef = { + id: string; + label: string; + entity_type: string; + aliases: string[]; +}; + +export type SmritiConfig = { + version: number; + categories?: CustomCategoryDef[]; + entities?: CustomEntityDef[]; + allowedCategories?: string[]; + autoSync?: boolean; +}; + +// ============================================================================= +// Read / Write +// ============================================================================= + +const CONFIG_FILE = "config.json"; + +export function readConfig(smritiDir: string): SmritiConfig { + try { + const { readFileSync } = require("fs"); + const json = readFileSync(join(smritiDir, CONFIG_FILE), "utf-8"); + return JSON.parse(json) as SmritiConfig; + } catch { + return { version: 1 }; + } +} + +export async function writeConfig( + smritiDir: string, + config: SmritiConfig +): Promise { + await Bun.write(join(smritiDir, CONFIG_FILE), JSON.stringify(config, null, 2)); +} + +// ============================================================================= +// Category Merge +// ============================================================================= + +const BUILTIN_IDS = new Set(ALL_CATEGORY_IDS); + +/** + * Upsert custom categories from config into the local DB. + * Sorts by slash-depth so parents are always created before children. + * Returns count of newly created categories. + */ +export function mergeCategories( + db: Database, + categories: CustomCategoryDef[] +): number { + if (categories.length === 0) return 0; + + // Parents before children: sort by number of slashes in id + const sorted = [...categories].sort( + (a, b) => (a.id.split("/").length) - (b.id.split("/").length) + ); + + let created = 0; + for (const cat of sorted) { + if (BUILTIN_IDS.has(cat.id)) continue; + const existing = db + .prepare(`SELECT id FROM smriti_categories WHERE id = ?`) + .get(cat.id); + if (existing) continue; + + // Validate parent exists if specified + if (cat.parent) { + const parentExists = db + .prepare(`SELECT id FROM smriti_categories WHERE id = ?`) + .get(cat.parent); + if (!parentExists) continue; // skip orphan — parent not yet created + } + + db.prepare( + `INSERT OR IGNORE INTO smriti_categories (id, name, parent_id, description) + VALUES (?, ?, ?, ?)` + ).run(cat.id, cat.name, cat.parent ?? null, cat.description ?? null); + created++; + } + return created; +} + +// ============================================================================= +// Export custom categories from DB +// ============================================================================= + +/** + * Query smriti_categories for non-builtin entries and return as config defs. + */ +export function exportCustomCategories(db: Database): CustomCategoryDef[] { + const rows = db + .prepare(`SELECT id, name, parent_id, description FROM smriti_categories`) + .all() as { id: string; name: string; parent_id: string | null; description: string | null }[]; + + return rows + .filter((r) => !BUILTIN_IDS.has(r.id)) + .map((r) => ({ + id: r.id, + name: r.name, + ...(r.parent_id ? { parent: r.parent_id } : {}), + ...(r.description ? { description: r.description } : {}), + })); +} + +// ============================================================================= +// Entity Merge — team/org propagation for smriti_entities +// +// Same reasoning as categories: an entity's canonical id is only meaningful +// if every teammate's machine agrees on it. .smriti/config.json is the +// git-committed source of truth each local smriti_entities table converges +// toward on every share/sync — the same role a published vocabulary plays +// for literal RDF, minus the URIs. +// ============================================================================= + +/** + * Upsert entities from config into the local DB. Matches first by id, then + * falls back to a normalized-label match (so two machines that independently + * minted different ids for the same concept still converge once either + * syncs the shared file). Unions aliases on conflict rather than overwriting. + * Returns count of newly created entities. + */ +export function mergeEntities(db: Database, entities: CustomEntityDef[]): number { + if (entities.length === 0) return 0; + + let created = 0; + for (const entity of entities) { + const existingById = db + .prepare(`SELECT id, aliases FROM smriti_entities WHERE id = ?`) + .get(entity.id) as { id: string; aliases: string } | null; + + if (existingById) { + const aliases = new Set(JSON.parse(existingById.aliases)); + for (const a of entity.aliases) aliases.add(a); + db.prepare(`UPDATE smriti_entities SET aliases = ? WHERE id = ?`) + .run(JSON.stringify([...aliases]), entity.id); + continue; + } + + const normalizedLabel = entity.label.trim().toLowerCase(); + const existingByLabel = db + .prepare(`SELECT id, aliases FROM smriti_entities WHERE LOWER(label) = ?`) + .get(normalizedLabel) as { id: string; aliases: string } | null; + + if (existingByLabel) { + const aliases = new Set(JSON.parse(existingByLabel.aliases)); + for (const a of entity.aliases) aliases.add(a); + db.prepare(`UPDATE smriti_entities SET aliases = ? WHERE id = ?`) + .run(JSON.stringify([...aliases]), existingByLabel.id); + continue; + } + + db.prepare( + `INSERT INTO smriti_entities (id, label, entity_type, aliases, mention_count) + VALUES (?, ?, ?, ?, 0)` + ).run(entity.id, entity.label, entity.entity_type, JSON.stringify(entity.aliases)); + created++; + } + return created; +} + +/** Query smriti_entities and return as config defs for export to .smriti/config.json. */ +export function exportEntities(db: Database): CustomEntityDef[] { + const rows = db + .prepare(`SELECT id, label, entity_type, aliases FROM smriti_entities`) + .all() as Array<{ id: string; label: string; entity_type: string; aliases: string }>; + + return rows.map((r) => ({ + id: r.id, + label: r.label, + entity_type: r.entity_type, + aliases: JSON.parse(r.aliases), + })); +} diff --git a/src/team/ollama.ts b/src/team/ollama.ts index 6ea04f7..655347c 100644 --- a/src/team/ollama.ts +++ b/src/team/ollama.ts @@ -5,7 +5,7 @@ * Used by segment.ts (Stage 1) and document.ts (Stage 2). */ -import { OLLAMA_HOST, OLLAMA_MODEL } from "../config"; +import { OLLAMA_HOST, requireOllamaModel } from "../config"; export type OllamaOptions = { model?: string; @@ -28,7 +28,7 @@ export async function callOllama( prompt: string, options: OllamaOptions = {} ): Promise { - const model = options.model || OLLAMA_MODEL; + const model = requireOllamaModel(options.model); const temperature = options.temperature ?? 0.7; const timeout = options.timeout ?? DEFAULT_TIMEOUT; const maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES; diff --git a/src/team/reflect.ts b/src/team/reflect.ts index 32ca106..6612709 100644 --- a/src/team/reflect.ts +++ b/src/team/reflect.ts @@ -10,7 +10,7 @@ * 2. src/team/prompts/share-reflect.md (built-in default) */ -import { OLLAMA_HOST, OLLAMA_MODEL } from "../config"; +import { OLLAMA_HOST, requireOllamaModel } from "../config"; import { join, dirname } from "path"; import { fileURLToPath } from "url"; import type { RawMessage } from "./formatter"; @@ -222,7 +222,7 @@ export async function synthesizeSession( const template = await loadPromptTemplate(options.projectSmritiDir); const prompt = template.replace("{{conversation}}", conversation); - const model = options.model || OLLAMA_MODEL; + const model = requireOllamaModel(options.model); const timeout = options.timeout || 120_000; const controller = new AbortController(); diff --git a/src/team/share.ts b/src/team/share.ts index 49894fd..22adef8 100644 --- a/src/team/share.ts +++ b/src/team/share.ts @@ -10,6 +10,7 @@ import { SMRITI_DIR, AUTHOR } from "../config"; import { hashContent } from "../qmd"; import { existsSync, mkdirSync } from "fs"; import { join } from "path"; +import { readConfig, writeConfig, exportCustomCategories, exportEntities } from "./config"; import { formatSessionAsFallback, isSessionWorthSharing, @@ -139,7 +140,7 @@ function querySessions( } /** Get messages for a session */ -function getSessionMessages( +export function getSessionMessages( db: Database, sessionId: string ): Array<{ @@ -162,7 +163,8 @@ function getSessionMessages( /** Write manifest and config files, generate CLAUDE.md */ async function writeManifest( outputDir: string, - newEntries: Array<{ id: string; category: string; file: string; shared_at: string }> + newEntries: Array<{ id: string; category: string; file: string; shared_at: string }>, + db?: Database ): Promise { const indexPath = join(outputDir, "index.json"); let existingManifest: any[] = []; @@ -176,22 +178,19 @@ async function writeManifest( const fullManifest = [...existingManifest, ...newEntries]; await Bun.write(indexPath, JSON.stringify(fullManifest, null, 2)); - // Write config if it doesn't exist - const configPath = join(outputDir, "config.json"); - if (!existsSync(configPath)) { - await Bun.write( - configPath, - JSON.stringify( - { - version: 1, - allowedCategories: ["*"], - autoSync: false, - }, - null, - 2 - ) - ); - } + // Write config — always update with latest custom categories + canonical entities + const existing = readConfig(outputDir); + const customCategories = db ? exportCustomCategories(db) : []; + const entities = db ? exportEntities(db) : []; + const config = { + ...existing, + version: customCategories.length > 0 || entities.length > 0 ? 2 : (existing.version ?? 1), + allowedCategories: existing.allowedCategories ?? ["*"], + autoSync: existing.autoSync ?? false, + ...(customCategories.length > 0 ? { categories: customCategories } : {}), + ...(entities.length > 0 ? { entities } : {}), + }; + await writeConfig(outputDir, config); // Generate CLAUDE.md await generateClaudeMd(outputDir, fullManifest); @@ -351,7 +350,7 @@ async function shareSegmentedKnowledge( } } - await writeManifest(outputDir, manifest); + await writeManifest(outputDir, manifest, db); return result; } @@ -517,7 +516,7 @@ export async function shareKnowledge( } } - await writeManifest(outputDir, manifest); + await writeManifest(outputDir, manifest, db); return result; } diff --git a/src/team/sync.ts b/src/team/sync.ts index a0612a5..b4e51cc 100644 --- a/src/team/sync.ts +++ b/src/team/sync.ts @@ -9,6 +9,8 @@ import type { Database } from "bun:sqlite"; import { SMRITI_DIR } from "../config"; import { addMessage, hashContent } from "../qmd"; import { join } from "path"; +import { readConfig, mergeCategories, mergeEntities } from "./config"; +import { insertRelationship, type RelationshipPredicate } from "../learn/entities"; // ============================================================================= // Types @@ -24,6 +26,8 @@ export type SyncResult = { imported: number; skipped: number; errors: string[]; + categoriesImported: number; + entitiesImported: number; }; // ============================================================================= @@ -32,22 +36,27 @@ export type SyncResult = { /** Parse YAML frontmatter from a markdown file */ export function parseFrontmatter(content: string): { - meta: Record; + meta: Record; body: string; } { const match = content.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/); if (!match) return { meta: {}, body: content }; - const meta: Record = {}; + const meta: Record = {}; for (const line of match[1].split("\n")) { const colonIdx = line.indexOf(":"); if (colonIdx > 0) { const key = line.slice(0, colonIdx).trim(); - const value = line - .slice(colonIdx + 1) - .trim() - .replace(/^["']|["']$/g, ""); - meta[key] = value; + const raw = line.slice(colonIdx + 1).trim(); + if (raw.startsWith("[") && raw.endsWith("]")) { + meta[key] = raw + .slice(1, -1) + .split(",") + .map((s) => s.trim().replace(/^["']|["']$/g, "")) + .filter(Boolean); + } else { + meta[key] = raw.replace(/^["']|["']$/g, ""); + } } } @@ -108,6 +117,8 @@ export async function syncTeamKnowledge( imported: 0, skipped: 0, errors: [], + categoriesImported: 0, + entitiesImported: 0, }; // Determine input directory @@ -136,6 +147,17 @@ export async function syncTeamKnowledge( ).map((r) => r.content_hash) ); + // Import custom categories + canonical entities from config.json (v2+) + // before scanning files — entities must exist locally before per-file + // "mentions"/relationship edges below can reference them. + const config = readConfig(inputDir); + if (config.categories && config.categories.length > 0) { + result.categoriesImported = mergeCategories(db, config.categories); + } + if (config.entities && config.entities.length > 0) { + result.entitiesImported = mergeEntities(db, config.entities); + } + // Scan for markdown files const knowledgeDir = join(inputDir, "knowledge"); const glob = new Bun.Glob("**/*.md"); @@ -161,10 +183,11 @@ export async function syncTeamKnowledge( continue; } - // Segmented pipeline docs don't have **user**/**assistant** patterns; - // treat the whole body as a single assistant message. - const isSegmented = meta.pipeline === "segmented"; - const messages = isSegmented + // Segmented and consolidated pipeline docs don't have + // **user**/**assistant** patterns; treat the whole body as a single + // assistant message. + const isSingleMessageDoc = meta.pipeline === "segmented" || meta.pipeline === "consolidated"; + const messages = isSingleMessageDoc ? [{ role: "assistant", content: body.trim() }] : extractMessages(body); @@ -173,9 +196,13 @@ export async function syncTeamKnowledge( continue; } + // Helper: coerce meta field to plain string + const metaStr = (v: string | string[] | undefined): string => + Array.isArray(v) ? (v[0] ?? "") : (v ?? ""); + // Create session from the imported file const sessionId = - meta.id || `team-${crypto.randomUUID().slice(0, 8)}`; + metaStr(meta.id) || `team-${crypto.randomUUID().slice(0, 8)}`; // Extract title from heading const titleMatch = body.match(/^#\s+(.+)/m); @@ -189,28 +216,63 @@ export async function syncTeamKnowledge( upsertSessionMeta( db, sessionId, - meta.agent || "team", - meta.project || options.project + metaStr(meta.agent) || "team", + metaStr(meta.project) || options.project ); - // Apply category tags - if (meta.category) { - tagSession(db, sessionId, meta.category, 1.0, "team"); + // Restore all tags from the tags array; fall back to scalar category + const { isValidCategory } = await import("../categorize/schema"); + if (Array.isArray(meta.tags) && meta.tags.length > 0) { + for (const tag of meta.tags) { + if (isValidCategory(db, tag)) { + tagSession(db, sessionId, tag, 1.0, "team"); + } + } + } else if (meta.category) { + const cat = metaStr(meta.category); + if (isValidCategory(db, cat)) { + tagSession(db, sessionId, cat, 1.0, "team"); + } } // Record the share for dedup + const primaryCategory = Array.isArray(meta.tags) + ? (meta.tags[0] ?? metaStr(meta.category) ?? null) + : metaStr(meta.category) || null; db.prepare( `INSERT OR IGNORE INTO smriti_shares (id, session_id, category_id, project_id, author, content_hash) VALUES (?, ?, ?, ?, ?, ?)` ).run( crypto.randomUUID().slice(0, 8), sessionId, - meta.category || null, - meta.project || null, - meta.author || "team", + primaryCategory, + metaStr(meta.project) || null, + metaStr(meta.author) || "team", contentHash ); + // Re-create relationship edges from frontmatter. Unlike entities, + // these need no canonicalization: unit ids are portable UUIDs + // already shared as `sessionId` above, so edges reference the same + // node on every machine that imports this file. + const asArray = (v: string | string[] | undefined): string[] => + v === undefined ? [] : Array.isArray(v) ? v : [v]; + + for (const entityId of asArray(meta.entity_ids)) { + if (!entityId) continue; + insertRelationship(db, "knowledge_unit", sessionId, "mentions", "entity", entityId, { + source: "extraction", + }); + } + for (const predicate of ["relatesTo", "supersedes", "contradicts"] as RelationshipPredicate[]) { + for (const objectId of asArray(meta[predicate])) { + if (!objectId) continue; + insertRelationship(db, "knowledge_unit", sessionId, predicate, "knowledge_unit", objectId, { + source: "llm", + }); + } + } + result.imported++; } catch (err: any) { result.errors.push(`${match}: ${err.message}`); diff --git a/test/cursor-sqlite.test.ts b/test/cursor-sqlite.test.ts new file mode 100644 index 0000000..ee6417d --- /dev/null +++ b/test/cursor-sqlite.test.ts @@ -0,0 +1,393 @@ +/** + * cursor-sqlite.test.ts + * + * Tests for the Cursor SQLite-based ingest path. + * Creates fixture SQLite databases in a temp dir — never touches the real home dir. + */ +import { test, expect, beforeEach, afterEach } from "bun:test"; +import { Database } from "bun:sqlite"; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "fs"; +import { join } from "path"; +import { tmpdir } from "os"; + +import { + resolveComposerMessages, + discoverCursorSqliteSessions, + buildComposerWorkspaceMap, + parseCursorJson, +} from "../src/ingest/cursor"; + +import { initializeMemoryTables } from "../src/qmd"; +import { initializeSmritiTables, seedDefaults } from "../src/db"; +import { ingest } from "../src/ingest/index"; + +// ============================================================================= +// Helpers +// ============================================================================= + +function createGlobalDb(path: string) { + const db = new Database(path); + db.exec(` + CREATE TABLE IF NOT EXISTS ItemTable (key TEXT PRIMARY KEY, value TEXT); + CREATE TABLE IF NOT EXISTS cursorDiskKV (key TEXT PRIMARY KEY, value TEXT); + `); + return db; +} + +function createWorkspaceDb(path: string) { + const db = new Database(path); + db.exec(`CREATE TABLE IF NOT EXISTS ItemTable (key TEXT PRIMARY KEY, value TEXT);`); + return db; +} + +function insertComposer( + db: Database, + composerId: string, + data: Record +) { + db.prepare( + `INSERT OR REPLACE INTO cursorDiskKV (key, value) VALUES (?, ?)` + ).run(`composerData:${composerId}`, JSON.stringify({ composerId, ...data })); +} + +function insertBubble( + db: Database, + composerId: string, + bubbleId: string, + type: number, + text: string, + createdAt?: string +) { + const data: Record = { type, bubbleId, text }; + if (createdAt) data.createdAt = createdAt; + db.prepare( + `INSERT OR REPLACE INTO cursorDiskKV (key, value) VALUES (?, ?)` + ).run(`bubbleId:${composerId}:${bubbleId}`, JSON.stringify(data)); +} + +// ============================================================================= +// resolveComposerMessages +// ============================================================================= + +test("resolveComposerMessages: inline conversation (older format)", () => { + const composer = { + composerId: "comp1", + conversation: [ + { bubbleId: "b1", type: 1, text: "Hello from user" }, + { bubbleId: "b2", type: 2, text: "Hello back" }, + { bubbleId: "b3", type: 1, text: "" }, // empty — skip + ], + }; + const msgs = resolveComposerMessages(composer, new Map()); + expect(msgs.length).toBe(2); + expect(msgs[0].role).toBe("user"); + expect(msgs[0].content).toBe("Hello from user"); + expect(msgs[1].role).toBe("assistant"); + expect(msgs[1].content).toBe("Hello back"); +}); + +test("resolveComposerMessages: fullConversationHeadersOnly + bubble lookup (newer format)", () => { + const bubbleMap = new Map([ + ["b1", { type: 1, text: "User question", bubbleId: "b1" }], + ["b2", { type: 2, text: "Assistant answer", bubbleId: "b2", createdAt: "2025-11-20T10:00:00Z" }], + ]); + const composer = { + composerId: "comp2", + fullConversationHeadersOnly: [ + { bubbleId: "b1", type: 1 }, + { bubbleId: "b2", type: 2 }, + ], + }; + const msgs = resolveComposerMessages(composer, bubbleMap); + expect(msgs.length).toBe(2); + expect(msgs[0].role).toBe("user"); + expect(msgs[0].content).toBe("User question"); + expect(msgs[1].role).toBe("assistant"); + expect(msgs[1].content).toBe("Assistant answer"); + expect(msgs[1].timestamp).toBe("2025-11-20T10:00:00Z"); +}); + +test("resolveComposerMessages: inline conversation takes priority over headers", () => { + const composer = { + composerId: "comp3", + conversation: [ + { bubbleId: "b1", type: 1, text: "From inline" }, + ], + fullConversationHeadersOnly: [ + { bubbleId: "b99", type: 1 }, // would return nothing from empty map + ], + }; + const msgs = resolveComposerMessages(composer, new Map()); + expect(msgs.length).toBe(1); + expect(msgs[0].content).toBe("From inline"); +}); + +test("resolveComposerMessages: skips bubbles not in map", () => { + const bubbleMap = new Map([ + ["b1", { type: 1, text: "Present bubble", bubbleId: "b1" }], + ]); + const composer = { + composerId: "comp4", + fullConversationHeadersOnly: [ + { bubbleId: "b1", type: 1 }, + { bubbleId: "b_missing", type: 2 }, // not in map + ], + }; + const msgs = resolveComposerMessages(composer, bubbleMap); + expect(msgs.length).toBe(1); + expect(msgs[0].content).toBe("Present bubble"); +}); + +test("resolveComposerMessages: skips unknown type numbers", () => { + const composer = { + composerId: "comp5", + conversation: [ + { bubbleId: "b1", type: 99, text: "Unknown type" }, // type 99 = skip + { bubbleId: "b2", type: 1, text: "Valid user" }, + ], + }; + const msgs = resolveComposerMessages(composer, new Map()); + expect(msgs.length).toBe(1); + expect(msgs[0].content).toBe("Valid user"); +}); + +// ============================================================================= +// discoverCursorSqliteSessions +// ============================================================================= + +let tmpDir: string; +let globalDb: Database; +let globalDbPath: string; + +beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), "smriti-cursor-test-")); + globalDbPath = join(tmpDir, "global.vscdb"); + globalDb = createGlobalDb(globalDbPath); +}); + +afterEach(() => { + globalDb.close(); + try { + rmSync(tmpDir, { recursive: true, force: true }); + } catch { + // Windows: SQLite file-lock release can lag close(), making rm throw + // EBUSY. The dir is under tmpdir() on an ephemeral runner — leak is fine. + } +}); + +test("discoverCursorSqliteSessions: fullConversationHeadersOnly with 2 bubbles", () => { + const composerId = "test-composer-headers"; + insertComposer(globalDb, composerId, { + createdAt: 1700000000000, + fullConversationHeadersOnly: [ + { bubbleId: "b1", type: 1 }, + { bubbleId: "b2", type: 2 }, + ], + }); + insertBubble(globalDb, composerId, "b1", 1, "What is TypeScript?"); + insertBubble(globalDb, composerId, "b2", 2, "TypeScript is a typed superset of JavaScript."); + globalDb.close(); + + const sessions = discoverCursorSqliteSessions(globalDbPath, new Map()); + expect(sessions.length).toBe(1); + expect(sessions[0].meta.sessionId).toBe(`cursor-${composerId}`); + expect(sessions[0].messages.length).toBe(2); + expect(sessions[0].messages[0].role).toBe("user"); + expect(sessions[0].messages[1].role).toBe("assistant"); + expect(sessions[0].meta.createdAt).toBe(new Date(1700000000000).toISOString()); + + // Re-open for afterEach cleanup + globalDb = new Database(globalDbPath); +}); + +test("discoverCursorSqliteSessions: inline conversation format", () => { + const composerId = "test-composer-inline"; + insertComposer(globalDb, composerId, { + createdAt: 1700000001000, + conversation: [ + { bubbleId: "b1", type: 1, text: "Fix my bug please" }, + { bubbleId: "b2", type: 2, text: "I can fix it." }, + ], + }); + globalDb.close(); + + const sessions = discoverCursorSqliteSessions(globalDbPath, new Map()); + expect(sessions.length).toBe(1); + expect(sessions[0].messages.length).toBe(2); + expect(sessions[0].messages[0].content).toBe("Fix my bug please"); + + globalDb = new Database(globalDbPath); +}); + +test("discoverCursorSqliteSessions: skips composers with no messages", () => { + insertComposer(globalDb, "empty-comp", { + createdAt: 1700000002000, + fullConversationHeadersOnly: [], // no headers -> no messages + }); + insertComposer(globalDb, "null-value-comp", { + createdAt: 1700000003000, + conversation: [ + { bubbleId: "b1", type: 1, text: "" }, // empty text -> skipped + ], + }); + globalDb.close(); + + const sessions = discoverCursorSqliteSessions(globalDbPath, new Map()); + expect(sessions.length).toBe(0); + + globalDb = new Database(globalDbPath); +}); + +test("discoverCursorSqliteSessions: filters by projectPath", () => { + const composer1 = "comp-proj-a"; + const composer2 = "comp-proj-b"; + + insertComposer(globalDb, composer1, { + createdAt: 1700000004000, + conversation: [{ bubbleId: "b1", type: 1, text: "For project A" }], + }); + insertComposer(globalDb, composer2, { + createdAt: 1700000005000, + conversation: [{ bubbleId: "b2", type: 1, text: "For project B" }], + }); + globalDb.close(); + + const composerMap = new Map([ + [composer1, "/Users/test/project-a"], + [composer2, "/Users/test/project-b"], + ]); + const sessions = discoverCursorSqliteSessions(globalDbPath, composerMap, { + projectPath: "/Users/test/project-a", + }); + expect(sessions.length).toBe(1); + expect(sessions[0].meta.sessionId).toBe(`cursor-${composer1}`); + + globalDb = new Database(globalDbPath); +}); + +test("discoverCursorSqliteSessions: maps projectPath from composerWorkspaceMap", () => { + const composerId = "comp-with-project"; + insertComposer(globalDb, composerId, { + createdAt: 1700000006000, + conversation: [{ bubbleId: "b1", type: 1, text: "Question" }], + }); + globalDb.close(); + + const composerMap = new Map([[composerId, "/Users/test/my-project"]]); + const sessions = discoverCursorSqliteSessions(globalDbPath, composerMap); + expect(sessions.length).toBe(1); + expect(sessions[0].meta.projectPath).toBe("/Users/test/my-project"); + + globalDb = new Database(globalDbPath); +}); + +// ============================================================================= +// buildComposerWorkspaceMap +// ============================================================================= + +test("buildComposerWorkspaceMap: reads composerId from workspace state.vscdb", () => { + // Setup: /workspaceStorage// + const wsStorageDir = join(tmpDir, "workspaceStorage"); + const hashDir = join(wsStorageDir, "abc123"); + mkdirSync(hashDir, { recursive: true }); + + // workspace.json + writeFileSync( + join(hashDir, "workspace.json"), + JSON.stringify({ folder: "file:///Users/test/my-app" }) + ); + + // state.vscdb + const wsDb = createWorkspaceDb(join(hashDir, "state.vscdb")); + wsDb.prepare( + `INSERT INTO ItemTable (key, value) VALUES (?, ?)` + ).run( + "composer.composerData", + JSON.stringify({ + allComposers: [ + { composerId: "aaa-111", createdAt: 0 }, + { composerId: "bbb-222", createdAt: 0 }, + ], + }) + ); + wsDb.close(); + + const map = buildComposerWorkspaceMap(tmpDir); + expect(map.get("aaa-111")).toBe("/Users/test/my-app"); + expect(map.get("bbb-222")).toBe("/Users/test/my-app"); +}); + +test("buildComposerWorkspaceMap: handles missing or malformed DBs gracefully", () => { + // workspaceStorage dir exists but has no valid DBs + const wsStorageDir = join(tmpDir, "workspaceStorage"); + mkdirSync(join(wsStorageDir, "bad_hash"), { recursive: true }); + // no state.vscdb, no workspace.json + const map = buildComposerWorkspaceMap(tmpDir); + expect(map.size).toBe(0); +}); + +// ============================================================================= +// Legacy parseCursorJson (backward compat) +// ============================================================================= + +test("parseCursorJson: still works for legacy .cursor/*.json format", () => { + const json = JSON.stringify({ + messages: [ + { role: "user", content: "Legacy question" }, + { role: "assistant", content: "Legacy answer" }, + ], + }); + const msgs = parseCursorJson(json); + expect(msgs.length).toBe(2); + expect(msgs[0].role).toBe("user"); + expect(msgs[1].role).toBe("assistant"); +}); + +// ============================================================================= +// End-to-end: ingest(cursor) via orchestrator with fixture SQLite DB +// ============================================================================= + +test("ingest(cursor) ingests SQLite sessions without projectPath", async () => { + // We need a full Cursor user dir structure: + // /Cursor/User/globalStorage/state.vscdb + // /Cursor/User/workspaceStorage//state.vscdb + workspace.json + const cursorUserRoot = join(tmpDir, "Cursor", "User"); + const globalStorageDir = join(cursorUserRoot, "globalStorage"); + mkdirSync(globalStorageDir, { recursive: true }); + + const gPath = join(globalStorageDir, "state.vscdb"); + const gDb = createGlobalDb(gPath); + const composerId = "e2e-composer-1"; + insertComposer(gDb, composerId, { + createdAt: 1700000010000, + conversation: [ + { bubbleId: "b1", type: 1, text: "End to end user message" }, + { bubbleId: "b2", type: 2, text: "End to end assistant message" }, + ], + }); + gDb.close(); + + // Override env to point to our test Cursor root + const origEnv = Bun.env.CURSOR_STORAGE_DIR; + Bun.env.CURSOR_STORAGE_DIR = cursorUserRoot; + + // In-memory Smriti DB + const memDb = new Database(":memory:"); + memDb.exec("PRAGMA foreign_keys = ON"); + const { initializeMemoryTables } = await import("../src/qmd"); + const { initializeSmritiTables, seedDefaults } = await import("../src/db"); + initializeMemoryTables(memDb); + initializeSmritiTables(memDb); + seedDefaults(memDb); + + const result = await ingest(memDb, "cursor"); + + Bun.env.CURSOR_STORAGE_DIR = origEnv; + memDb.close(); + + expect(result.agent).toBe("cursor"); + expect(result.sessionsFound).toBeGreaterThanOrEqual(1); + expect(result.sessionsIngested).toBeGreaterThanOrEqual(1); + expect(result.messagesIngested).toBeGreaterThanOrEqual(2); + expect(result.errors).toHaveLength(0); +}); diff --git a/test/daemon-client.test.ts b/test/daemon-client.test.ts new file mode 100644 index 0000000..0e9bff8 --- /dev/null +++ b/test/daemon-client.test.ts @@ -0,0 +1,102 @@ +/** + * test/daemon-client.test.ts + * + * Unit tests for the daemon lifecycle client (getDaemonStatus + stopDaemon). + * Tests work through the PID file rather than spawning real subprocesses. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; + +import { DAEMON_PID_FILE, DAEMON_SOCKET_FILE } from "../src/config"; +import { getDaemonStatus, stopDaemon } from "../src/daemon/client"; + +function cleanupDaemonState() { + try { rmSync(DAEMON_PID_FILE, { force: true }); } catch {} + try { rmSync(DAEMON_SOCKET_FILE, { force: true }); } catch {} +} + +function ensureCacheDir() { + mkdirSync(dirname(DAEMON_PID_FILE), { recursive: true }); +} + +describe("getDaemonStatus", () => { + beforeEach(cleanupDaemonState); + afterEach(cleanupDaemonState); + + test("returns not-running when no PID file exists", () => { + const s = getDaemonStatus(); + expect(s.running).toBe(false); + expect(s.pid).toBeNull(); + expect(s.startedAt).toBeNull(); + expect(s.pidFile).toBe(DAEMON_PID_FILE); + }); + + test("returns not-running and cleans up when PID file is stale", () => { + ensureCacheDir(); + writeFileSync(DAEMON_PID_FILE, "999999"); + const s = getDaemonStatus(); + expect(s.running).toBe(false); + // detectRunningDaemon should have removed the stale file + expect(existsSync(DAEMON_PID_FILE)).toBe(false); + }); + + test("returns running with PID and startedAt when our own PID is in the file", () => { + ensureCacheDir(); + writeFileSync(DAEMON_PID_FILE, String(process.pid)); + const s = getDaemonStatus(); + expect(s.running).toBe(true); + expect(s.pid).toBe(process.pid); + expect(s.startedAt).toBeInstanceOf(Date); + // PID file was just written; startedAt should be very recent. + expect(Date.now() - s.startedAt!.getTime()).toBeLessThan(2000); + }); +}); + +describe("stopDaemon", () => { + beforeEach(cleanupDaemonState); + afterEach(cleanupDaemonState); + + test("returns not-running when no daemon is running", async () => { + const r = await stopDaemon(); + expect(r.state).toBe("not-running"); + }); + + test("returns not-running when PID file holds a dead PID", async () => { + ensureCacheDir(); + writeFileSync(DAEMON_PID_FILE, "999999"); // dead pid; detectRunningDaemon cleans + returns null + const r = await stopDaemon(); + expect(r.state).toBe("not-running"); + }); + + // win32: there is no signal emulation — process.kill(pid, "SIGTERM") is + // TerminateProcess, so stopDaemon() SIGTERMing our own PID kills the test + // runner outright (no handler can swallow it). Daemon is unshipped on + // Windows; skip there. + test.skipIf(process.platform === "win32")("returns timeout when the daemon doesn't exit in the window", async () => { + // Pin our own PID in the file. We won't actually receive SIGTERM in + // this test process (the harness installs handlers), but the PID + // file will keep saying we're alive, so stopDaemon will time out. + ensureCacheDir(); + writeFileSync(DAEMON_PID_FILE, String(process.pid)); + + // Disable any SIGTERM handler we may have inherited, just for this test. + const sigtermListeners = process.listeners("SIGTERM"); + process.removeAllListeners("SIGTERM"); + // Reinstate a no-op so receiving SIGTERM doesn't kill the test runner. + const noop = () => {}; + process.on("SIGTERM", noop); + + try { + const r = await stopDaemon({ timeoutMs: 200, pollMs: 50 }); + expect(r.state).toBe("timeout"); + if (r.state === "timeout") { + expect(r.pid).toBe(process.pid); + } + } finally { + process.removeListener("SIGTERM", noop); + for (const l of sigtermListeners) process.on("SIGTERM", l as any); + } + }); +}); diff --git a/test/daemon-handlers.test.ts b/test/daemon-handlers.test.ts new file mode 100644 index 0000000..fbb0a6d --- /dev/null +++ b/test/daemon-handlers.test.ts @@ -0,0 +1,57 @@ +/** + * test/daemon-handlers.test.ts + * + * Unit tests for the FS-event → agent routing helpers. + */ + +import { describe, expect, test } from "bun:test"; +import { sep } from "node:path"; + +import { resolveAgentForPath, type AgentRoot } from "../src/daemon/handlers"; + +const roots: AgentRoot[] = [ + { agent: "claude", root: `${sep}home${sep}user${sep}.claude${sep}projects` }, + { agent: "codex", root: `${sep}home${sep}user${sep}.codex` }, + { agent: "cline", root: `${sep}home${sep}user${sep}.cline${sep}tasks` }, +]; + +describe("resolveAgentForPath", () => { + test("returns the agent when path is exactly the root", () => { + expect(resolveAgentForPath(roots[0].root, roots)).toBe("claude"); + }); + + test("returns the agent when path is under the root", () => { + expect( + resolveAgentForPath(`${roots[0].root}${sep}proj-a${sep}sess.jsonl`, roots), + ).toBe("claude"); + }); + + test("returns null for paths outside any root", () => { + expect(resolveAgentForPath(`${sep}tmp${sep}other.jsonl`, roots)).toBeNull(); + }); + + test("does not match by prefix-substring (e.g. .claude/projects-archive)", () => { + // A common bug shape: ".claude/projects" naively matching ".claude/projects-archive" + const sibling = `${sep}home${sep}user${sep}.claude${sep}projects-archive${sep}old.jsonl`; + expect(resolveAgentForPath(sibling, roots)).toBeNull(); + }); + + test("dispatches the right agent across multiple watched roots", () => { + expect( + resolveAgentForPath(`${roots[1].root}${sep}sessions${sep}s.jsonl`, roots), + ).toBe("codex"); + expect( + resolveAgentForPath(`${roots[2].root}${sep}task-1${sep}messages.json`, roots), + ).toBe("cline"); + }); + + test("ignores roots whose root string is empty (unconfigured agents)", () => { + const withEmpty: AgentRoot[] = [ + { agent: "copilot", root: "" }, + ...roots, + ]; + // Empty root must not match anything. + expect(resolveAgentForPath(roots[0].root, withEmpty)).toBe("claude"); + expect(resolveAgentForPath("", withEmpty)).toBeNull(); + }); +}); diff --git a/test/daemon-install.test.ts b/test/daemon-install.test.ts new file mode 100644 index 0000000..e2c3fc7 --- /dev/null +++ b/test/daemon-install.test.ts @@ -0,0 +1,240 @@ +/** + * test/daemon-install.test.ts + * + * Unit tests for the service-file installer. Tests the pure template + * generators against fixtures, and tests the install/uninstall flow + * against a mock RunCmd and a temp directory for the service path — + * so we don't actually register anything with the real launchctl / + * systemctl during CI. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; + +import { + SMRITI_LABEL, + generatePlist, + generateSystemdUnit, + installDaemon, + uninstallDaemon, + type InstallTarget, + type RunCmd, + type RunResult, +} from "../src/daemon/install"; + +// Helpers ------------------------------------------------------------ + +let tmpHome: string; +let darwinTarget: InstallTarget; +let linuxTarget: InstallTarget; + +beforeEach(() => { + tmpHome = mkdtempSync(join(tmpdir(), "smriti-install-test-")); + darwinTarget = { + platform: "darwin", + servicePath: join(tmpHome, "LaunchAgents", `${SMRITI_LABEL}.plist`), + }; + linuxTarget = { + platform: "linux", + servicePath: join(tmpHome, "systemd", "user", "smriti.service"), + }; +}); + +afterEach(() => { + try { rmSync(tmpHome, { recursive: true, force: true }); } catch {} +}); + +function recordingRunner(): { calls: { cmd: string; args: string[] }[]; run: RunCmd; nextResult: (r: RunResult) => void } { + const calls: { cmd: string; args: string[] }[] = []; + let queued: RunResult[] = []; + const run: RunCmd = async (cmd, args) => { + calls.push({ cmd, args }); + return queued.shift() ?? { code: 0, stdout: "", stderr: "" }; + }; + return { calls, run, nextResult: (r) => queued.push(r) }; +} + +const LAUNCH = { exec: "/usr/local/bin/bun", args: ["/path/to/smriti.js", "daemon"] }; + +// Template generators ------------------------------------------------ + +describe("generatePlist", () => { + test("emits a valid plist structure with the smriti label", () => { + const plist = generatePlist({ launch: LAUNCH, logFile: "/tmp/daemon.log" }); + expect(plist).toContain(`${SMRITI_LABEL}`); + expect(plist).toContain("RunAtLoad"); + expect(plist).toContain(""); + expect(plist).toContain("KeepAlive"); + expect(plist).toContain("/usr/local/bin/bun"); + expect(plist).toContain("/path/to/smriti.js"); + expect(plist).toContain("daemon"); + expect(plist).toContain("/tmp/daemon.log"); + }); + + test("escapes XML-special characters in paths", () => { + const plist = generatePlist({ + launch: { exec: "/path/with&\"quote", args: ["x"] }, + logFile: "/tmp/x", + }); + expect(plist).toContain("<gt>&amp;"quote"); + expect(plist).not.toMatch(/<\/string><[^/]/); // no broken XML + }); +}); + +describe("generateSystemdUnit", () => { + test("emits an ExecStart line with the launch target", () => { + const unit = generateSystemdUnit({ launch: LAUNCH }); + expect(unit).toContain("Description=Smriti daemon — cross-agent capture"); + expect(unit).toContain("ExecStart=/usr/local/bin/bun /path/to/smriti.js daemon"); + expect(unit).toContain("Restart=on-failure"); + expect(unit).toContain("WantedBy=default.target"); + }); + + test("quotes ExecStart args that need escaping", () => { + const unit = generateSystemdUnit({ + launch: { exec: "/usr/local/bin/bun", args: ["/path with spaces/main.js", "daemon"] }, + }); + expect(unit).toContain('ExecStart=/usr/local/bin/bun "/path with spaces/main.js" daemon'); + }); +}); + +// installDaemon ------------------------------------------------------ + +describe("installDaemon (macOS)", () => { + test("writes plist and invokes launchctl bootstrap", async () => { + const r = recordingRunner(); + const result = await installDaemon({ + target: darwinTarget, + launch: LAUNCH, + logFile: "/tmp/test.log", + run: r.run, + log: () => {}, + }); + + expect(result.wrote).toBe(true); + expect(existsSync(darwinTarget.servicePath)).toBe(true); + expect(readFileSync(darwinTarget.servicePath, "utf-8")).toContain(SMRITI_LABEL); + + expect(r.calls[0].cmd).toBe("launchctl"); + expect(r.calls[0].args[0]).toBe("bootstrap"); + expect(r.calls[0].args[2]).toBe(darwinTarget.servicePath); + }); + + test("treats already-loaded label as success, not error", async () => { + const r = recordingRunner(); + r.nextResult({ code: 17, stdout: "", stderr: "Bootstrap failed: 17: already loaded" }); + + const result = await installDaemon({ + target: darwinTarget, + launch: LAUNCH, + run: r.run, + log: () => {}, + }); + + expect(result.alreadyRegistered).toBe(true); + }); + + test("falls back to launchctl load when bootstrap fails with a non-EEXIST code", async () => { + const r = recordingRunner(); + r.nextResult({ code: 1, stdout: "", stderr: "Bootstrap failed: some other reason" }); + r.nextResult({ code: 0, stdout: "", stderr: "" }); + + await installDaemon({ + target: darwinTarget, + launch: LAUNCH, + run: r.run, + log: () => {}, + }); + + expect(r.calls[0].args[0]).toBe("bootstrap"); + expect(r.calls[1].cmd).toBe("launchctl"); + expect(r.calls[1].args[0]).toBe("load"); + }); + + test("idempotent: re-install with matching content skips rewrite", async () => { + const r1 = recordingRunner(); + await installDaemon({ target: darwinTarget, launch: LAUNCH, run: r1.run, log: () => {} }); + + const r2 = recordingRunner(); + const result = await installDaemon({ target: darwinTarget, launch: LAUNCH, run: r2.run, log: () => {} }); + + // File content matches, so it's not rewritten — but launchctl is still + // called (registration is the cheap idempotent operation). + expect(result.wrote).toBe(false); + expect(r2.calls.length).toBeGreaterThan(0); + }); +}); + +describe("installDaemon (Linux)", () => { + test("writes service file and runs daemon-reload + enable --now", async () => { + const r = recordingRunner(); + const result = await installDaemon({ + target: linuxTarget, + launch: LAUNCH, + run: r.run, + log: () => {}, + }); + + expect(result.wrote).toBe(true); + expect(existsSync(linuxTarget.servicePath)).toBe(true); + expect(readFileSync(linuxTarget.servicePath, "utf-8")).toContain("ExecStart="); + + expect(r.calls[0]).toEqual({ cmd: "systemctl", args: ["--user", "daemon-reload"] }); + expect(r.calls[1]).toEqual({ cmd: "systemctl", args: ["--user", "enable", "--now", "smriti"] }); + }); + + test("throws when systemctl daemon-reload fails", async () => { + const r = recordingRunner(); + r.nextResult({ code: 1, stdout: "", stderr: "no systemd" }); + + await expect( + installDaemon({ target: linuxTarget, launch: LAUNCH, run: r.run, log: () => {} }), + ).rejects.toThrow(/daemon-reload failed/); + }); +}); + +// uninstallDaemon ---------------------------------------------------- + +describe("uninstallDaemon (macOS)", () => { + test("removes plist and calls launchctl bootout", async () => { + // Pre-seed the service file + mkdirSync(dirname(darwinTarget.servicePath), { recursive: true }); + writeFileSync(darwinTarget.servicePath, "", { flag: "w" }); + + const r = recordingRunner(); + const result = await uninstallDaemon({ target: darwinTarget, run: r.run, log: () => {} }); + + expect(result.unregistered).toBe(true); + expect(result.removedFile).toBe(true); + expect(existsSync(darwinTarget.servicePath)).toBe(false); + + expect(r.calls[0].cmd).toBe("launchctl"); + expect(r.calls[0].args[0]).toBe("bootout"); + }); +}); + +describe("uninstallDaemon (Linux)", () => { + test("calls systemctl disable --now and removes service file", async () => { + mkdirSync(dirname(linuxTarget.servicePath), { recursive: true }); + writeFileSync(linuxTarget.servicePath, "[Unit]\n", { flag: "w" }); + + const r = recordingRunner(); + const result = await uninstallDaemon({ target: linuxTarget, run: r.run, log: () => {} }); + + expect(result.unregistered).toBe(true); + expect(result.removedFile).toBe(true); + expect(r.calls[0]).toEqual({ cmd: "systemctl", args: ["--user", "disable", "--now", "smriti"] }); + // daemon-reload after removal + expect(r.calls.some((c) => c.args.join(" ") === "--user daemon-reload")).toBe(true); + }); + + test("succeeds even when no service file exists", async () => { + const r = recordingRunner(); + const result = await uninstallDaemon({ target: linuxTarget, run: r.run, log: () => {} }); + expect(result.removedFile).toBe(false); + // systemctl disable is still attempted (it's idempotent and cheap) + expect(r.calls.length).toBeGreaterThan(0); + }); +}); diff --git a/test/daemon-queue.test.ts b/test/daemon-queue.test.ts new file mode 100644 index 0000000..68c5ef7 --- /dev/null +++ b/test/daemon-queue.test.ts @@ -0,0 +1,167 @@ +/** + * test/daemon-queue.test.ts + * + * Unit tests for the per-project debounce queue. Uses short debounce + * windows so the suite runs quickly. + */ + +import { describe, expect, test } from "bun:test"; +import { createDebounceQueue } from "../src/daemon/queue"; + +const settle = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +describe("createDebounceQueue", () => { + test("schedule + wait fires onFlush exactly once", async () => { + const calls: string[] = []; + const q = createDebounceQueue({ + debounceMs: 50, + onFlush: (id) => { calls.push(id); }, + }); + + q.schedule("proj-a"); + await settle(100); + + expect(calls).toEqual(["proj-a"]); + expect(q.pending()).toBe(0); + q.close(); + }); + + test("schedule N times within the window coalesces to one onFlush", async () => { + const calls: string[] = []; + const q = createDebounceQueue({ + debounceMs: 100, + onFlush: (id) => { calls.push(id); }, + }); + + q.schedule("proj-a"); + await settle(20); + q.schedule("proj-a"); + await settle(20); + q.schedule("proj-a"); + await settle(150); + + expect(calls).toEqual(["proj-a"]); + q.close(); + }); + + test("different projects have independent timers", async () => { + const calls: string[] = []; + const q = createDebounceQueue({ + debounceMs: 50, + onFlush: (id) => { calls.push(id); }, + }); + + q.schedule("proj-a"); + q.schedule("proj-b"); + q.schedule("proj-c"); + expect(q.pending()).toBe(3); + await settle(100); + + expect(calls.sort()).toEqual(["proj-a", "proj-b", "proj-c"]); + expect(q.pending()).toBe(0); + q.close(); + }); + + test("flush() fires immediately and cancels the debounce", async () => { + const calls: string[] = []; + const q = createDebounceQueue({ + debounceMs: 10_000, // very long; would never naturally fire + onFlush: (id) => { calls.push(id); }, + }); + + q.schedule("proj-a"); + expect(q.pending()).toBe(1); + await q.flush("proj-a"); + + expect(calls).toEqual(["proj-a"]); + expect(q.pending()).toBe(0); + q.close(); + }); + + test("flush() with no pending timer still fires onFlush", async () => { + const calls: string[] = []; + const q = createDebounceQueue({ + debounceMs: 100, + onFlush: (id) => { calls.push(id); }, + }); + + await q.flush("proj-fresh"); + + expect(calls).toEqual(["proj-fresh"]); + q.close(); + }); + + test("close() prevents pending timers from firing", async () => { + const calls: string[] = []; + const q = createDebounceQueue({ + debounceMs: 50, + onFlush: (id) => { calls.push(id); }, + }); + + q.schedule("proj-a"); + q.schedule("proj-b"); + q.close(); + await settle(100); + + expect(calls).toEqual([]); + }); + + test("close() prevents subsequent schedule() from registering", async () => { + const calls: string[] = []; + const q = createDebounceQueue({ + debounceMs: 50, + onFlush: (id) => { calls.push(id); }, + }); + + q.close(); + q.schedule("proj-a"); + await settle(100); + + expect(calls).toEqual([]); + expect(q.pending()).toBe(0); + }); + + test("onFlush errors are logged but don't crash the queue", async () => { + const logs: string[] = []; + const q = createDebounceQueue({ + debounceMs: 30, + onFlush: () => { throw new Error("simulated"); }, + log: (m) => logs.push(m), + }); + + q.schedule("proj-a"); + await settle(80); + + expect(logs.some((m) => m.includes("simulated"))).toBe(true); + + // Queue still works after the error. + const calls: string[] = []; + const q2 = createDebounceQueue({ + debounceMs: 30, + onFlush: (id) => { calls.push(id); }, + }); + q2.schedule("proj-b"); + await settle(80); + expect(calls).toEqual(["proj-b"]); + + q.close(); + q2.close(); + }); + + test("isPending() reflects per-project state", async () => { + const q = createDebounceQueue({ + debounceMs: 200, + onFlush: () => {}, + }); + + expect(q.isPending("proj-a")).toBe(false); + q.schedule("proj-a"); + expect(q.isPending("proj-a")).toBe(true); + expect(q.isPending("proj-b")).toBe(false); + + await q.flush("proj-a"); + expect(q.isPending("proj-a")).toBe(false); + + q.close(); + }); +}); diff --git a/test/daemon-runner.test.ts b/test/daemon-runner.test.ts new file mode 100644 index 0000000..4a6ffe0 --- /dev/null +++ b/test/daemon-runner.test.ts @@ -0,0 +1,205 @@ +/** + * test/daemon-runner.test.ts + * + * Integration tests for runDaemon() — verifies the wiring between + * watcher, queue, server, and the flushAgent callback. Uses a + * temporary directory as the agent root and a mock flushAgent so + * the real ingest path (and the user's real DB) are not touched. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { + mkdtempSync, + rmSync, + writeFileSync, + mkdirSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createConnection } from "node:net"; + +import { DAEMON_PID_FILE, DAEMON_SOCKET_FILE } from "../src/config"; +import { runDaemon } from "../src/daemon"; + +const settle = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +function cleanupDaemonState() { + try { rmSync(DAEMON_PID_FILE, { force: true }); } catch {} + try { rmSync(DAEMON_SOCKET_FILE, { force: true }); } catch {} +} + +describe("runDaemon", () => { + let tmpRoot: string; + + beforeEach(() => { + cleanupDaemonState(); + tmpRoot = mkdtempSync(join(tmpdir(), "smriti-runner-test-")); + }); + + afterEach(() => { + try { rmSync(tmpRoot, { recursive: true, force: true }); } catch {} + cleanupDaemonState(); + }); + + test("an FS event in a watched root triggers flushAgent after debounce", async () => { + const flushes: string[] = []; + const daemon = await runDaemon({ + agentRoots: [{ agent: "fake-agent", root: tmpRoot }], + flushAgent: (agent) => { flushes.push(agent); }, + debounceMs: 100, + log: () => {}, + }); + + await settle(50); // watcher attach + writeFileSync(join(tmpRoot, "session.jsonl"), "line\n"); + await settle(200); // debounce + flush + + expect(flushes).toEqual(["fake-agent"]); + await daemon.shutdown(); + }); + + test("rapid writes coalesce into one flush per project", async () => { + const flushes: string[] = []; + const daemon = await runDaemon({ + agentRoots: [{ agent: "fake-agent", root: tmpRoot }], + flushAgent: (agent) => { flushes.push(agent); }, + debounceMs: 150, + log: () => {}, + }); + + await settle(50); + for (let i = 0; i < 5; i++) { + writeFileSync(join(tmpRoot, `s${i}.jsonl`), "x"); + await settle(20); + } + await settle(250); // wait past the debounce window + + expect(flushes).toEqual(["fake-agent"]); + await daemon.shutdown(); + }); + + test("a poke on the IPC socket immediately flushes 'claude'", async () => { + const flushes: string[] = []; + const daemon = await runDaemon({ + agentRoots: [], + flushAgent: (agent) => { flushes.push(agent); }, + debounceMs: 60_000, // never naturally fire + log: () => {}, + }); + + await new Promise((resolve, reject) => { + const c = createConnection(DAEMON_SOCKET_FILE, () => { c.end(); }); + c.on("close", () => resolve()); + c.on("error", reject); + }); + await settle(80); + + expect(flushes).toEqual(["claude"]); + await daemon.shutdown(); + }); + + test("multiple agent roots route events to the correct agent", async () => { + const sub1 = join(tmpRoot, "a"); + const sub2 = join(tmpRoot, "b"); + mkdirSync(sub1); + mkdirSync(sub2); + + const flushes: string[] = []; + const daemon = await runDaemon({ + agentRoots: [ + { agent: "agent-A", root: sub1 }, + { agent: "agent-B", root: sub2 }, + ], + flushAgent: (agent) => { flushes.push(agent); }, + debounceMs: 80, + log: () => {}, + }); + + await settle(50); + writeFileSync(join(sub2, "from-b.jsonl"), "x"); + await settle(200); + + expect(flushes).toEqual(["agent-B"]); + await daemon.shutdown(); + }); + + test("flushAgent errors are isolated — the daemon keeps running", async () => { + const errors: string[] = []; + let firstCall = true; + const daemon = await runDaemon({ + agentRoots: [{ agent: "fake-agent", root: tmpRoot }], + flushAgent: () => { + if (firstCall) { + firstCall = false; + throw new Error("simulated ingest failure"); + } + }, + debounceMs: 80, + log: (m) => { errors.push(m); }, + }); + + await settle(50); + writeFileSync(join(tmpRoot, "first.jsonl"), "x"); + await settle(180); + writeFileSync(join(tmpRoot, "second.jsonl"), "x"); + await settle(180); + + expect(errors.some((m) => m.includes("simulated ingest failure"))).toBe(true); + // Daemon should still be alive — shutdown cleanly without timing out. + await daemon.shutdown(); + }); + + test("shutdown is idempotent and cleans up PID + socket", async () => { + const daemon = await runDaemon({ + agentRoots: [{ agent: "fake-agent", root: tmpRoot }], + flushAgent: () => {}, + debounceMs: 100, + log: () => {}, + }); + + await daemon.shutdown(); + await daemon.shutdown(); // should not throw + expect(true).toBe(true); // assertion presence + }); + + const ENRICH_ENV_KEY = "SMRITI_INGEST_NO_ENRICH"; + // Read via a non-literal key so TS doesn't (incorrectly) narrow this to + // `undefined` from the `delete` below — it can't see that runDaemon() + // mutates process.env internally. + const readEnrichFlag = () => process.env[ENRICH_ENV_KEY]; + + test("sets SMRITI_INGEST_NO_ENRICH by default so routine ingest never triggers LLM enrichment", async () => { + const original = readEnrichFlag(); + delete process.env[ENRICH_ENV_KEY]; + try { + const daemon = await runDaemon({ + agentRoots: [], + flushAgent: () => {}, + log: () => {}, + }); + expect(readEnrichFlag()).toBe("1"); + await daemon.shutdown(); + } finally { + if (original === undefined) delete process.env[ENRICH_ENV_KEY]; + else process.env[ENRICH_ENV_KEY] = original; + } + }); + + test("enrichOnIngest: true opts back into LLM enrichment during ingest", async () => { + const original = readEnrichFlag(); + delete process.env[ENRICH_ENV_KEY]; + try { + const daemon = await runDaemon({ + agentRoots: [], + flushAgent: () => {}, + log: () => {}, + enrichOnIngest: true, + }); + expect(readEnrichFlag()).toBeUndefined(); + await daemon.shutdown(); + } finally { + if (original === undefined) delete process.env[ENRICH_ENV_KEY]; + else process.env[ENRICH_ENV_KEY] = original; + } + }); +}); diff --git a/test/daemon-server.test.ts b/test/daemon-server.test.ts new file mode 100644 index 0000000..b966c9d --- /dev/null +++ b/test/daemon-server.test.ts @@ -0,0 +1,141 @@ +/** + * test/daemon-server.test.ts + * + * Unit tests for the daemon single-instance + lifecycle primitives. + * Focused on detectRunningDaemon() and the startDaemon() happy path. + * + * The IPC socket and ingest wiring is covered in higher-level tests. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; + +import { DAEMON_PID_FILE, DAEMON_SOCKET_FILE } from "../src/config"; +import { detectRunningDaemon, startDaemon } from "../src/daemon/server"; + +// Helpers ------------------------------------------------------------ + +function cleanupDaemonState() { + try { rmSync(DAEMON_PID_FILE, { force: true }); } catch {} + try { rmSync(DAEMON_SOCKET_FILE, { force: true }); } catch {} +} + +function ensureCacheDir() { + mkdirSync(dirname(DAEMON_PID_FILE), { recursive: true }); +} + +// detectRunningDaemon ----------------------------------------------- + +describe("detectRunningDaemon", () => { + beforeEach(cleanupDaemonState); + afterEach(cleanupDaemonState); + + test("returns null when no PID file exists", () => { + expect(detectRunningDaemon()).toBeNull(); + }); + + test("returns null and cleans up when PID file holds a dead PID", () => { + ensureCacheDir(); + // PID 1 is init/launchd on every Unix — we can't probe it without + // permission, but PID 0 is reserved and always reports ESRCH. Picking + // something high and unlikely: + const deadPid = 999999; + writeFileSync(DAEMON_PID_FILE, String(deadPid)); + expect(detectRunningDaemon()).toBeNull(); + expect(existsSync(DAEMON_PID_FILE)).toBe(false); // stale file cleaned + }); + + test("returns null and cleans up garbage PID files", () => { + ensureCacheDir(); + writeFileSync(DAEMON_PID_FILE, "not-a-number"); + expect(detectRunningDaemon()).toBeNull(); + expect(existsSync(DAEMON_PID_FILE)).toBe(false); + }); + + test("returns our own PID when the PID file points at us", () => { + ensureCacheDir(); + writeFileSync(DAEMON_PID_FILE, String(process.pid)); + expect(detectRunningDaemon()).toBe(process.pid); + // Our PID file should still be there — it's a live PID. + expect(existsSync(DAEMON_PID_FILE)).toBe(true); + }); + + test("returns null for an empty PID file", () => { + ensureCacheDir(); + writeFileSync(DAEMON_PID_FILE, ""); + expect(detectRunningDaemon()).toBeNull(); + expect(existsSync(DAEMON_PID_FILE)).toBe(false); + }); +}); + +// startDaemon ------------------------------------------------------- + +describe("startDaemon", () => { + beforeEach(cleanupDaemonState); + afterEach(cleanupDaemonState); + + // win32: AF_UNIX socket binding behaves differently and the daemon is + // unshipped on Windows — skip the real-socket test there. + test.skipIf(process.platform === "win32")("writes PID file and binds socket, shutdown() reverses both", async () => { + const logs: string[] = []; + const handle = await startDaemon({ log: (m) => logs.push(m) }); + + expect(handle.pid).toBe(process.pid); + expect(existsSync(DAEMON_PID_FILE)).toBe(true); + expect(existsSync(DAEMON_SOCKET_FILE)).toBe(true); + expect(readFileSync(DAEMON_PID_FILE, "utf-8").trim()).toBe(String(process.pid)); + + await handle.shutdown(); + + expect(existsSync(DAEMON_PID_FILE)).toBe(false); + expect(existsSync(DAEMON_SOCKET_FILE)).toBe(false); + }); + + test("throws when another live daemon owns the PID file", async () => { + ensureCacheDir(); + writeFileSync(DAEMON_PID_FILE, String(process.pid)); // pretend a live daemon + await expect(startDaemon({ log: () => {} })).rejects.toThrow(/already running/i); + // Our pre-seeded PID file should be untouched. + expect(existsSync(DAEMON_PID_FILE)).toBe(true); + }); + + test("succeeds and overwrites when the existing PID file is stale", async () => { + ensureCacheDir(); + writeFileSync(DAEMON_PID_FILE, "999999"); // dead pid + const handle = await startDaemon({ log: () => {} }); + expect(readFileSync(DAEMON_PID_FILE, "utf-8").trim()).toBe(String(process.pid)); + await handle.shutdown(); + }); + + test("shutdown() is idempotent", async () => { + const handle = await startDaemon({ log: () => {} }); + await handle.shutdown(); + await handle.shutdown(); // second call should be a no-op, not throw + expect(existsSync(DAEMON_PID_FILE)).toBe(false); + }); + + test("onPoke fires when a client connects to the socket", async () => { + let pokes = 0; + const handle = await startDaemon({ + log: () => {}, + onPoke: () => { pokes += 1; }, + }); + + // Connect via Bun's net client and immediately close. + const { createConnection } = await import("node:net"); + await new Promise((resolve, reject) => { + const conn = createConnection(DAEMON_SOCKET_FILE, () => { + conn.end(); + }); + conn.on("close", () => resolve()); + conn.on("error", reject); + }); + + // Allow the poke handler to run (it's invoked from the connection callback). + await new Promise((r) => setTimeout(r, 50)); + expect(pokes).toBe(1); + + await handle.shutdown(); + }); +}); diff --git a/test/daemon-watcher.test.ts b/test/daemon-watcher.test.ts new file mode 100644 index 0000000..1f0cf21 --- /dev/null +++ b/test/daemon-watcher.test.ts @@ -0,0 +1,135 @@ +/** + * test/daemon-watcher.test.ts + * + * Unit tests for watchRecursive(). Uses temporary directories so they're + * deterministic and don't depend on the user's real Claude/Codex state. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { watchRecursive, type WatcherEvent } from "../src/daemon/watcher"; + +// Helpers ------------------------------------------------------------ + +let tmpRoot: string; + +beforeEach(() => { + tmpRoot = mkdtempSync(join(tmpdir(), "smriti-watcher-test-")); +}); + +afterEach(() => { + try { rmSync(tmpRoot, { recursive: true, force: true }); } catch {} +}); + +/** Wait briefly for fs.watch events to flush. macOS FSEvents has a small delay. */ +const settle = (ms = 250) => new Promise((r) => setTimeout(r, ms)); + +// Tests -------------------------------------------------------------- + +// The daemon is deferred on Windows (see release notes); Bun's fs.watch on +// win32 hard-crashes the test process, so skip the suite there entirely. +describe.skipIf(process.platform === "win32")("watchRecursive", () => { + test("throws if root does not exist", () => { + expect(() => + watchRecursive(join(tmpRoot, "does-not-exist"), () => {}), + ).toThrow(/root does not exist/); + }); + + test("fires events for direct-child file creation", async () => { + const events: WatcherEvent[] = []; + const w = watchRecursive(tmpRoot, (e) => events.push(e)); + + await settle(50); // give the watcher a moment to attach + writeFileSync(join(tmpRoot, "a.txt"), "hello"); + await settle(); + + expect(events.length).toBeGreaterThan(0); + expect(events.some((e) => e.path.endsWith("a.txt"))).toBe(true); + + w.close(); + }); + + test("fires events for files in subdirectories (recursive)", async () => { + const sub = join(tmpRoot, "subA", "subB"); + mkdirSync(sub, { recursive: true }); + + const events: WatcherEvent[] = []; + const w = watchRecursive(tmpRoot, (e) => events.push(e)); + await settle(50); + + writeFileSync(join(sub, "deep.txt"), "hello"); + await settle(); + + expect(events.some((e) => e.path.endsWith("deep.txt"))).toBe(true); + w.close(); + }); + + test("fires events for content changes (not just creation)", async () => { + const file = join(tmpRoot, "log.jsonl"); + writeFileSync(file, "first\n"); + + const events: WatcherEvent[] = []; + const w = watchRecursive(tmpRoot, (e) => events.push(e)); + await settle(50); + + writeFileSync(file, "first\nsecond\n"); + await settle(); + + // Either 'rename' or 'change' is acceptable depending on backend + // (macOS FSEvents tends to fire 'rename' even for content edits in + // some cases). The important thing is *something* fired for this path. + expect(events.some((e) => e.path.endsWith("log.jsonl"))).toBe(true); + w.close(); + }); + + test("emits absolute paths, not relative ones", async () => { + const events: WatcherEvent[] = []; + const w = watchRecursive(tmpRoot, (e) => events.push(e)); + await settle(50); + + writeFileSync(join(tmpRoot, "abs.txt"), "x"); + await settle(); + + expect(events.length).toBeGreaterThan(0); + for (const e of events) { + expect(e.path.startsWith("/")).toBe(true); + } + w.close(); + }); + + test("close() stops further events", async () => { + const events: WatcherEvent[] = []; + const w = watchRecursive(tmpRoot, (e) => events.push(e)); + await settle(50); + + w.close(); + + writeFileSync(join(tmpRoot, "after-close.txt"), "x"); + await settle(); + + expect(events.find((e) => e.path.endsWith("after-close.txt"))).toBeUndefined(); + }); + + test("watchedCount() reflects the watcher topology", async () => { + // On macOS/Windows: native recursive = exactly 1 watcher. + // On Linux: walk-and-watch produces one watcher per directory. + mkdirSync(join(tmpRoot, "a", "b"), { recursive: true }); + mkdirSync(join(tmpRoot, "c")); + + const w = watchRecursive(tmpRoot, () => {}); + const count = w.watchedCount(); + + if (process.platform === "darwin" || process.platform === "win32") { + expect(count).toBe(1); + } else { + // Linux: root + a + a/b + c = 4 dirs + expect(count).toBeGreaterThanOrEqual(4); + } + + w.close(); + expect(w.watchedCount()).toBe(0); + }); +}); diff --git a/test/eval/fixtures/auth-migration.ts b/test/eval/fixtures/auth-migration.ts new file mode 100644 index 0000000..696c9e1 --- /dev/null +++ b/test/eval/fixtures/auth-migration.ts @@ -0,0 +1,58 @@ +import type { RecallScenario } from "./types"; + +/** Cross-session recall over time, plus precision against a same-project, same-vocabulary distractor. */ +export const AUTH_MIGRATION: RecallScenario = { + name: "auth: JWT -> session cookie migration, recalled later", + project: "webapp", + sessions: [ + { + id: "auth-s1", + daysAgo: 14, + messages: [ + { role: "user", content: "We're getting complaints that JWTs can't be revoked on logout. Should we move to session cookies?" }, + { role: "assistant", content: "Session cookies with server-side storage let you revoke instantly. JWTs need a blocklist, which defeats the point of being stateless." }, + { role: "user", content: "Let's switch. Use server-side session cookies, drop JWT for auth entirely." }, + { role: "assistant", content: "Decision: replaced JWT-based session tokens with server-side session cookies stored in Redis, keyed by session id, TTL 24h." }, + ], + }, + { + id: "auth-s2", + daysAgo: 10, + messages: [ + { role: "user", content: "Session cookie migration is live. Any edge cases with mobile clients that can't set cookies?" }, + { role: "assistant", content: "Mobile WebViews sometimes block third-party cookies; we added a fallback bearer-token header for the mobile app only." }, + ], + }, + { + id: "auth-s3-distractor", + daysAgo: 30, + messages: [ + { role: "user", content: "What's our password hashing algorithm for the auth system?" }, + { role: "assistant", content: "We use bcrypt with cost factor 12 for password hashing, unrelated to session management." }, + ], + }, + ], + probes: [ + { + // FTS5 MATCH is an AND of all terms within one message — terms must + // be literally present, not natural-language phrasing (see msg2: + // "Use server-side session cookies, drop JWT for auth entirely."). + query: "JWT session cookies auth", + description: "core decision recall — must surface the migration session", + expectHitSessionIds: ["auth-s1"], + expectHitSubstrings: ["server-side session cookies"], + expectMissSessionIds: ["auth-s3-distractor"], + }, + { + query: "mobile cookie migration", + description: "follow-up detail in a later session — recall must surface s2, not just s1", + expectHitSessionIds: ["auth-s2"], + }, + { + query: "password hashing algorithm", + description: "distractor probe — different topic sharing the 'auth' vocabulary; must not pull in s1/s2", + expectHitSessionIds: ["auth-s3-distractor"], + expectMissSessionIds: ["auth-s1", "auth-s2"], + }, + ], +}; diff --git a/test/eval/fixtures/density-recency.ts b/test/eval/fixtures/density-recency.ts new file mode 100644 index 0000000..1e3acc4 --- /dev/null +++ b/test/eval/fixtures/density-recency.ts @@ -0,0 +1,41 @@ +import type { RecallScenario } from "./types"; + +/** + * Two sessions with identical content (so their BM25/RRF scores tie exactly) + * but very different density_score — recallMemories blends density into the + * final score 80/20, so at topK=1 only the denser session should survive. + * Requires useRecallMemories: true, since the project-filtered searchFiltered + * path never touches density scoring. + */ +export const DENSITY_BLENDING: RecallScenario = { + name: "density blending breaks a BM25 tie", + project: "backend", + sessions: [ + { + id: "density-high", + densityScore: 0.9, + messages: [ + { role: "user", content: "We should switch to using a message queue for background job processing." }, + { role: "assistant", content: "Agreed — moving long-running work off the request path avoids timeouts." }, + ], + }, + { + id: "density-low", + densityScore: 0.05, + messages: [ + { role: "user", content: "We should switch to using a message queue for background job processing." }, + { role: "assistant", content: "Agreed — moving long-running work off the request path avoids timeouts." }, + ], + }, + ], + probes: [ + { + query: "message queue background job processing", + description: "BM25-tied content, density_score must break the tie toward the denser session", + expectHitSessionIds: ["density-high"], + expectMissSessionIds: ["density-low"], + topK: 1, + useRecallMemories: true, + }, + ], +}; diff --git a/test/eval/fixtures/deploy-pipeline.ts b/test/eval/fixtures/deploy-pipeline.ts new file mode 100644 index 0000000..2065e9d --- /dev/null +++ b/test/eval/fixtures/deploy-pipeline.ts @@ -0,0 +1,42 @@ +import type { RecallScenario } from "./types"; + +/** + * Single-session recall, scoped by project. A second session shares almost + * identical vocabulary ("deploys are flaky", "CI pipeline") but lives under + * a different project — the probe (scoped to "api-service") must not pull + * it in, proving project filtering isolates results rather than just + * favoring topical relevance. + */ +export const DEPLOY_PIPELINE: RecallScenario = { + name: "deploy: CI pipeline decision, isolated by project", + project: "api-service", + sessions: [ + { + id: "deploy-s1", + messages: [ + { role: "user", content: "Our deploys are flaky. What's causing the intermittent CI pipeline failures?" }, + { role: "assistant", content: "Flaky tests were racing against a shared test database. Switched the pipeline to spin up an isolated Postgres container per CI job." }, + { role: "user", content: "Good, let's also cache node_modules between runs to speed things up." }, + { role: "assistant", content: "Added actions/cache keyed on the lockfile hash — pipeline runtime dropped from 8 minutes to 3." }, + ], + }, + { + id: "deploy-s2-other-project", + project: "frontend-app", + messages: [ + { role: "user", content: "Our deploys are flaky too — the CI pipeline times out on the frontend build." }, + { role: "assistant", content: "The frontend bundle got too large for the default Vercel build timeout; raised it and split the vendor chunk." }, + ], + }, + ], + probes: [ + { + // FTS5 MATCH is an AND of all terms within one message — literal terms only. + query: "CI pipeline flaky", + description: "single-session recall scoped to api-service — must not pull in the same-vocabulary frontend-app session", + expectHitSessionIds: ["deploy-s1"], + expectHitSubstrings: ["isolated Postgres container", "shared test database"], + expectMissSessionIds: ["deploy-s2-other-project"], + }, + ], +}; diff --git a/test/eval/fixtures/index.ts b/test/eval/fixtures/index.ts new file mode 100644 index 0000000..0db32bc --- /dev/null +++ b/test/eval/fixtures/index.ts @@ -0,0 +1,16 @@ +import { AUTH_MIGRATION } from "./auth-migration"; +import { DEPLOY_PIPELINE } from "./deploy-pipeline"; +import { DENSITY_BLENDING } from "./density-recency"; +import { SEMANTIC_CACHING } from "./semantic-caching"; +import type { RecallScenario } from "./types"; + +/** BM25-only scenarios — deterministic, no embeddings needed. Safe for CI. */ +export const CI_SCENARIOS: RecallScenario[] = [AUTH_MIGRATION, DEPLOY_PIPELINE, DENSITY_BLENDING]; + +/** Needs a live embedding backend — manual-only (see test/eval/recall-quality.eval.ts). */ +export const QUALITY_ONLY_SCENARIOS: RecallScenario[] = [SEMANTIC_CACHING]; + +/** Full set — quality mode runs all of these; CI mode filters to CI_SCENARIOS. */ +export const ALL_SCENARIOS: RecallScenario[] = [...CI_SCENARIOS, ...QUALITY_ONLY_SCENARIOS]; + +export type { RecallScenario, FixtureSession, Probe } from "./types"; diff --git a/test/eval/fixtures/run.ts b/test/eval/fixtures/run.ts new file mode 100644 index 0000000..04f59f0 --- /dev/null +++ b/test/eval/fixtures/run.ts @@ -0,0 +1,27 @@ +/** + * test/eval/fixtures/run.ts - Shared probe runner for the recall-quality + * harness (Tier 1 CI test and Tier 2 manual eval both call this). + */ + +import type { Database } from "bun:sqlite"; +import { recall } from "../../../src/search/recall"; +import { scoreProbe, type ProbeScore } from "./score"; +import type { Probe, RecallScenario } from "./types"; + +export const DEFAULT_TOP_K = 5; + +export async function runProbe( + db: Database, + scenario: RecallScenario, + probe: Probe, + options: { fast: boolean } +): Promise<{ score: ProbeScore; latencyMs: number; sources: string[] }> { + const limit = probe.topK ?? DEFAULT_TOP_K; + const started = performance.now(); + const { results } = probe.useRecallMemories + ? await recall(db, probe.query, { fast: options.fast, limit }) + : await recall(db, probe.query, { project: probe.project ?? scenario.project, fast: options.fast, limit }); + const latencyMs = performance.now() - started; + const score = scoreProbe(results, probe); + return { score, latencyMs, sources: [...new Set(results.map((r) => r.source))] }; +} diff --git a/test/eval/fixtures/score.ts b/test/eval/fixtures/score.ts new file mode 100644 index 0000000..3ba2535 --- /dev/null +++ b/test/eval/fixtures/score.ts @@ -0,0 +1,61 @@ +/** + * test/eval/fixtures/score.ts - Scoring for recall-quality probes. + * + * Precision is computed only against a probe's explicit expectMissSessionIds + * (known distractors), not exhaustively against everything else — the same + * "narrow but honest" approach test/eval/relation-inference.eval.ts uses for + * exact-match grading. We can't exhaustively label every session a probe + * shouldn't match, so we only assert on the ones we deliberately planted. + */ + +import type { Probe } from "./types"; + +export type ProbeResult = { session_id: string; content: string }; + +export type ProbeScore = { + recall: number; // |expected ∩ hit| / |expected|, 1 if no expected hits defined + precision: number | null; // 1 - |miss ∩ hit| / |hit|, null if the probe defines no distractors + substringOk: boolean; // true if no expectHitSubstrings, or one matched a retrieved expected-hit row + pass: boolean; +}; + +export function scoreProbe(results: ProbeResult[], probe: Probe): ProbeScore { + const hitIds = new Set(results.map((r) => r.session_id)); + + const expected = probe.expectHitSessionIds; + const hitCount = expected.filter((id) => hitIds.has(id)).length; + const recall = expected.length ? hitCount / expected.length : 1; + + let precision: number | null = null; + if (probe.expectMissSessionIds && probe.expectMissSessionIds.length > 0) { + const missHits = probe.expectMissSessionIds.filter((id) => hitIds.has(id)).length; + precision = hitIds.size > 0 ? 1 - missHits / hitIds.size : 1; + } + + let substringOk = true; + if (probe.expectHitSubstrings && probe.expectHitSubstrings.length > 0) { + const expectedRows = results.filter((r) => expected.includes(r.session_id)); + substringOk = expectedRows.some((r) => + probe.expectHitSubstrings!.some((s) => r.content.toLowerCase().includes(s.toLowerCase())) + ); + } + + const pass = recall === 1 && (precision === null || precision === 1) && substringOk; + return { recall, precision, substringOk, pass }; +} + +export function summarizeScores(scores: ProbeScore[]): { + total: number; + passed: number; + avgRecall: number; + avgPrecision: number | null; +} { + const total = scores.length; + const passed = scores.filter((s) => s.pass).length; + const avgRecall = total ? scores.reduce((sum, s) => sum + s.recall, 0) / total : 1; + const withPrecision = scores.filter((s) => s.precision !== null); + const avgPrecision = withPrecision.length + ? withPrecision.reduce((sum, s) => sum + (s.precision as number), 0) / withPrecision.length + : null; + return { total, passed, avgRecall, avgPrecision }; +} diff --git a/test/eval/fixtures/seed.ts b/test/eval/fixtures/seed.ts new file mode 100644 index 0000000..5f7c124 --- /dev/null +++ b/test/eval/fixtures/seed.ts @@ -0,0 +1,28 @@ +/** + * test/eval/fixtures/seed.ts - Seed a RecallScenario's sessions directly into + * a Smriti DB, bypassing real agent-log parsing entirely (mirrors the + * seedSession() helper in test/learn-consolidate.test.ts). + */ + +import type { Database } from "bun:sqlite"; +import { addMessage } from "../../../src/qmd"; +import { upsertProject, upsertSessionMeta, updateDensityScore } from "../../../src/db"; +import type { RecallScenario } from "./types"; + +const DAY_MS = 24 * 60 * 60 * 1000; + +export async function seedScenario(db: Database, scenario: RecallScenario): Promise { + const projects = new Set([scenario.project, ...scenario.sessions.map((s) => s.project ?? scenario.project)]); + for (const project of projects) upsertProject(db, project); + + for (const session of scenario.sessions) { + const createdAt = new Date(Date.now() - (session.daysAgo ?? 0) * DAY_MS).toISOString(); + for (const m of session.messages) { + await addMessage(db as any, session.id, m.role, m.content, { timestamp: createdAt }); + } + upsertSessionMeta(db, session.id, "claude-code", session.project ?? scenario.project); + if (session.densityScore !== undefined) { + updateDensityScore(db, session.id, session.densityScore); + } + } +} diff --git a/test/eval/fixtures/semantic-caching.ts b/test/eval/fixtures/semantic-caching.ts new file mode 100644 index 0000000..3dd4a6b --- /dev/null +++ b/test/eval/fixtures/semantic-caching.ts @@ -0,0 +1,40 @@ +import type { RecallScenario } from "./types"; + +/** + * Tier 2 only: the probe shares almost no lexical overlap with the source + * session (no "Redis", "cache", or "TTL" in the query) — only a real + * embedding model can bridge "in-memory data store to speed up repeated + * lookups" to "Redis... avoid repeated database round trips". + */ +export const SEMANTIC_CACHING: RecallScenario = { + name: "semantic-only match: caching decision, paraphrased query", + project: "api-service", + requiresEmbeddings: true, + sessions: [ + { + id: "semantic-s1", + messages: [ + { role: "user", content: "The product listing endpoint is slow under load." }, + { role: "assistant", content: "We store frequently accessed data in Redis to avoid repeated database round trips — added a 5-minute TTL for the product listing query." }, + ], + }, + { + id: "semantic-s2-distractor", + messages: [ + { role: "user", content: "Let's add rate limiting to the public API." }, + { role: "assistant", content: "Token bucket, 100 requests per minute per API key." }, + ], + }, + ], + probes: [ + { + query: "why do we use an in-memory data store to speed up repeated lookups", + description: "paraphrased, near-zero lexical overlap with the source session — needs semantic (vector) matching", + expectHitSessionIds: ["semantic-s1"], + expectMissSessionIds: ["semantic-s2-distractor"], + // The project-filtered path never touches vectors — route through + // recallMemories's hybrid pipeline, the only path embeddings affect. + useRecallMemories: true, + }, + ], +}; diff --git a/test/eval/fixtures/types.ts b/test/eval/fixtures/types.ts new file mode 100644 index 0000000..10fa61c --- /dev/null +++ b/test/eval/fixtures/types.ts @@ -0,0 +1,60 @@ +/** + * test/eval/fixtures/types.ts - Shared fixture format for the recall-quality + * harness. A scenario is a small multi-session conversation (optionally + * spanning sessions created at different points in time, via `daysAgo`) + * paired with probes that check both recall (the right session surfaces) + * and precision (a near-topic distractor does not). + * + * Shared by test/recall-quality.test.ts (Tier 1, CI-safe, BM25-only) and + * test/eval/recall-quality.eval.ts (Tier 2, manual, requires embeddings). + */ + +export type FixtureMessage = { role: "user" | "assistant"; content: string }; + +export type FixtureSession = { + id: string; + /** Overrides the scenario's default project — lets one scenario seed sessions across multiple projects to test project-filter isolation. */ + project?: string; + /** How many days before "now" this session was created — exercises recency/density blending. Omit for "just now". */ + daysAgo?: number; + /** density_score to set on this session (0-1). Omit to leave at the default (0). */ + densityScore?: number; + messages: FixtureMessage[]; +}; + +export type Probe = { + query: string; + /** Why a human would ask this — shown in the report, not asserted on. */ + description: string; + /** Session ids that MUST appear in the top-K results. */ + expectHitSessionIds: string[]; + /** At least one of these substrings must appear in a retrieved row belonging to an expected-hit session. */ + expectHitSubstrings?: string[]; + /** Session ids that must NOT appear in the top-K results (precision). */ + expectMissSessionIds?: string[]; + /** Defaults to the harness-wide DEFAULT_TOP_K. */ + topK?: number; + /** Overrides the scenario's default project for this probe's recall() call. */ + project?: string; + /** + * Route through recallMemories's unfiltered hybrid pipeline (RRF + density + * blending) instead of the project-filtered searchFiltered path. Still + * CI-safe without embeddings — vector search silently no-ops when none + * exist. Needed for probes that specifically exercise density/recency + * blending, which the project-filtered path never touches. + */ + useRecallMemories?: boolean; +}; + +export type RecallScenario = { + name: string; + /** Default project for sessions/probes that don't override it. */ + project: string; + sessions: FixtureSession[]; + probes: Probe[]; + /** + * Only runs in Tier 2 (manual, quality mode) — needs a live embedding + * backend to exercise genuinely non-lexical (semantic-only) matches. + */ + requiresEmbeddings?: boolean; +}; diff --git a/test/eval/recall-quality.eval.ts b/test/eval/recall-quality.eval.ts new file mode 100644 index 0000000..11c1196 --- /dev/null +++ b/test/eval/recall-quality.eval.ts @@ -0,0 +1,101 @@ +/** + * test/eval/recall-quality.eval.ts - Tier 2 of the recall-quality harness: + * the full fixture set from test/eval/fixtures/, including scenarios that + * need a live embedding backend to exercise genuinely semantic (non-lexical) + * matches — the CI-safe BM25-only subset already runs automatically as + * test/recall-quality.test.ts. + * + * NOT a bun:test file (no *.test.ts suffix) — needs a live embedding model + * (local llama.cpp, already a dependency, or Ollama via QMD_MEMORY_MODEL) and + * is slower than the CI subset, so it's excluded from `bun test` and run + * manually: + * + * bun run test/eval/recall-quality.eval.ts + */ + +import { initSmriti, closeDb } from "../../src/db"; +import { embedMemoryMessages } from "../../src/qmd"; +import { ALL_SCENARIOS } from "./fixtures/index"; +import { seedScenario } from "./fixtures/seed"; +import { runProbe, DEFAULT_TOP_K } from "./fixtures/run"; +import { summarizeScores, type ProbeScore } from "./fixtures/score"; +import type { RecallScenario } from "./fixtures/types"; + +type ProbeRun = { scenario: RecallScenario; query: string; description: string; score: ProbeScore; latencyMs: number; usedVectors: boolean }; + +async function runScenario(scenario: RecallScenario): Promise { + const db = await initSmriti(":memory:"); + const runs: ProbeRun[] = []; + try { + await seedScenario(db, scenario); + + let embedded = 0; + try { + embedded = await embedMemoryMessages(db as any); + } catch (err: any) { + console.log(` [warn] embedMemoryMessages failed for "${scenario.name}": ${err.message}`); + } + if (scenario.requiresEmbeddings && embedded === 0) { + console.log(` [warn] "${scenario.name}" needs embeddings but none were generated — results below may be BM25-only.`); + } + + for (const probe of scenario.probes) { + const { score, latencyMs, sources } = await runProbe(db, scenario, probe, { fast: false }); + runs.push({ + scenario, + query: probe.query, + description: probe.description, + score, + latencyMs, + usedVectors: sources.includes("vec"), + }); + } + } finally { + await closeDb(); + } + return runs; +} + +async function main() { + const embeddingScenarioCount = ALL_SCENARIOS.filter((s) => s.requiresEmbeddings).length; + console.log(`Running ${ALL_SCENARIOS.length} scenarios (${embeddingScenarioCount} require embeddings)...\n`); + + const allRuns: ProbeRun[] = []; + + for (const scenario of ALL_SCENARIOS) { + console.log(`## ${scenario.name}${scenario.requiresEmbeddings ? " (requires embeddings)" : ""}`); + const runs = await runScenario(scenario); + allRuns.push(...runs); + + for (const r of runs) { + const status = r.score.pass ? "PASS" : "FAIL"; + const vecTag = scenario.requiresEmbeddings ? (r.usedVectors ? " [vec]" : " [vec DID NOT FIRE]") : ""; + console.log( + ` [${status}] "${r.query}" — recall=${r.score.recall.toFixed(2)} precision=${r.score.precision === null ? "n/a" : r.score.precision.toFixed(2)} substrings=${r.score.substringOk}${vecTag} (${r.latencyMs.toFixed(0)}ms)` + ); + if (!r.score.pass) console.log(` ${r.description}`); + } + console.log(); + } + + const summary = summarizeScores(allRuns.map((r) => r.score)); + const vectorScenarios = allRuns.filter((r) => r.scenario.requiresEmbeddings); + const vectorsFired = vectorScenarios.filter((r) => r.usedVectors).length; + + console.log("=".repeat(60)); + console.log("SUMMARY"); + console.log("=".repeat(60)); + console.log(`Probes graded: ${summary.total}`); + console.log(`Probes passed: ${summary.passed}/${summary.total}`); + console.log(`Avg recall: ${summary.avgRecall.toFixed(2)}`); + console.log(`Avg precision: ${summary.avgPrecision === null ? "n/a" : summary.avgPrecision.toFixed(2)}`); + console.log(`Top-K: ${DEFAULT_TOP_K} (per-probe override via topK)`); + if (vectorScenarios.length > 0) { + console.log(`Vector search fired: ${vectorsFired}/${vectorScenarios.length} embedding-dependent probes`); + if (vectorsFired < vectorScenarios.length) { + console.log(` -> some embedding-dependent probes silently fell back to BM25-only. Check that a local embedding model or Ollama is reachable.`); + } + } +} + +await main(); diff --git a/test/eval/relation-inference.eval.ts b/test/eval/relation-inference.eval.ts new file mode 100644 index 0000000..010e576 --- /dev/null +++ b/test/eval/relation-inference.eval.ts @@ -0,0 +1,237 @@ +/** + * test/eval/relation-inference.eval.ts - Live A/B eval for relationship classification + * + * Compares classifyRelationshipsTextFormat ("before" — free-text RELATION + * lines parsed with a regex) against classifyRelationshipsToolCall ("after" + * — native tool calling) on a hand-labeled dataset, run against the real + * configured Ollama model (QMD_MEMORY_MODEL). + * + * NOT a bun:test file (no *.test.ts suffix) — it hits a live Ollama server + * and is slow/non-deterministic, so it's excluded from `bun test` and run + * manually: + * + * bun run test/eval/relation-inference.eval.ts + */ + +import { + classifyRelationshipsTextFormat, + classifyRelationshipsToolCall, + type RelationCandidate, + type RelationGuess, +} from "../../src/learn/consolidate"; + +// ============================================================================= +// Dataset +// ============================================================================= + +type Scenario = { + name: string; + unit: { topic: string; category: string; plainText: string }; + candidates: RelationCandidate[]; + expected: Array; // one expected predicate per candidate index +}; + +const SCENARIOS: Scenario[] = [ + { + name: "supersedes (reverted retrieval strategy)", + unit: { + topic: "Post-filtering for vector search", + category: "architecture/decision", + plainText: + "Switched the recall pipeline from pre-filtering to post-filtering with 3x overfetch, because pre-filtering caused sqlite-vec to hang when combined with JOINs on metadata tables.", + }, + candidates: [ + { id: "c1", topic: "Pre-filtering for vector search", category: "architecture/decision", plain_text: "Decision: use pre-filtering — apply metadata filters directly inside the sqlite-vec query before ranking." }, + ], + expected: ["supersedes"], + }, + { + name: "contradicts (daemon enrichment safety)", + unit: { + topic: "Inline LLM enrichment on daemon flush", + category: "architecture/decision", + plainText: "Team decided synchronous LLM enrichment on every daemon flush is safe and should run inline with ingestion.", + }, + candidates: [ + { id: "c1", topic: "Daemon flush safety", category: "architecture/decision", plain_text: "Decision: LLM enrichment must never run inline with daemon flush — it blocks ingestion and risks corrupting the write path under load." }, + ], + expected: ["contradicts"], + }, + { + name: "relatesTo (complementary recall features)", + unit: { + topic: "Cluster-scoped recall", + category: "feature/implementation", + plainText: "Added a --cluster flag to `smriti recall` for topic-scoped retrieval using O(1) Set membership checks.", + }, + candidates: [ + { id: "c1", topic: "RRF for recall", category: "feature/implementation", plain_text: "Implemented reciprocal rank fusion (RRF) to combine BM25 and vector search results in `smriti recall`." }, + ], + expected: ["relatesTo"], + }, + { + name: "none (unrelated domains)", + unit: { + topic: "Blog hover overlay CSS bug", + category: "bug/fix", + plainText: "Fixed a CSS bug where the hover overlay button showed a stray `.bv-tag` element on blog post cards.", + }, + candidates: [ + { id: "c1", topic: "Content hashing for dedup", category: "architecture/decision", plain_text: "Chose SHA256 content-addressable hashing for deduplicating ingested messages in QMD's content table." }, + ], + expected: ["none"], + }, + { + name: "supersedes (auth mechanism reversal)", + unit: { + topic: "Session cookies for auth", + category: "architecture/decision", + plainText: "Reverted from JWT-based session tokens to server-side session cookies after security review flagged JWT revocation as unsupported.", + }, + candidates: [ + { id: "c1", topic: "JWT session tokens", category: "architecture/decision", plain_text: "Adopted JWT-based session tokens for stateless auth across services." }, + ], + expected: ["supersedes"], + }, + { + name: "contradicts (MLX engine routing)", + unit: { + topic: "MLX engine routing in Ollama", + category: "topic/learning", + plainText: "Confirmed that `ollama pull` for MLX-tagged models requires model names ending in `-mlx`; regular GGUF pulls never use the MLX engine.", + }, + candidates: [ + { id: "c1", topic: "MLX engine routing in Ollama", category: "topic/learning", plain_text: "Established that any GGUF model pulled via `ollama pull` automatically runs on the MLX engine on Apple Silicon." }, + ], + expected: ["contradicts"], + }, + { + name: "relatesTo (ollama runner history)", + unit: { + topic: "Ollama runner architecture", + category: "topic/learning", + plainText: "Documented that Ollama's new llama-server-based runner replaced the old Go --ollama-engine runner starting in a later 0.x release.", + }, + candidates: [ + { id: "c1", topic: "Ollama runner architecture", category: "topic/learning", plain_text: "Verified Ollama 0.18 uses the ggml/Metal engine via a custom Go runner (--ollama-engine), not MLX, for standard GGUF models." }, + ], + expected: ["relatesTo"], + }, + { + name: "none (video upload vs tool-calling investigation)", + unit: { + topic: "Video metadata capture timeout", + category: "bug/fix", + plainText: "Fixed timeout and cancel handling in captureVideoMetadata to stop hangs during community photo/video upload.", + }, + candidates: [ + { id: "c1", topic: "MLX tool calling", category: "topic/learning", plain_text: "Investigated whether qwen3.5:9b-mlx-tuned supports native tool calling; confirmed via a /api/chat request with a get_weather tool schema." }, + ], + expected: ["none"], + }, + { + name: "multi-candidate index alignment", + unit: { + topic: "Standardize on post-filtering", + category: "architecture/decision", + plainText: "Standardized on post-filtering with 3x overfetch for all vector search filtering across projects.", + }, + candidates: [ + { id: "c1", topic: "Pre-filtering for vector search", category: "architecture/decision", plain_text: "Decision: use pre-filtering — apply metadata filters directly inside the sqlite-vec query." }, + { id: "c2", topic: "RRF for recall", category: "feature/implementation", plain_text: "Implemented RRF to merge BM25 and vector search scores." }, + { id: "c3", topic: "Blog hover overlay CSS bug", category: "bug/fix", plain_text: "Fixed a CSS bug in blog post hover overlays." }, + ], + expected: ["supersedes", "relatesTo", "none"], + }, +]; + +// ============================================================================= +// Runner +// ============================================================================= + +type MethodResult = { + guesses: RelationGuess[]; + latencyMs: number; + error?: string; +}; + +async function runMethod( + fn: (unit: Scenario["unit"], candidates: RelationCandidate[], model?: string) => Promise, + scenario: Scenario +): Promise { + const start = performance.now(); + try { + const guesses = await fn(scenario.unit, scenario.candidates); + return { guesses, latencyMs: performance.now() - start }; + } catch (err: any) { + return { guesses: [], latencyMs: performance.now() - start, error: err.message }; + } +} + +function grade(scenario: Scenario, guesses: RelationGuess[]) { + const byIndex = new Map(guesses.map((g) => [g.index, g.predicate])); + return scenario.expected.map((expected, index) => ({ + index, + expected, + got: byIndex.get(index) ?? "(missing)", + correct: byIndex.get(index) === expected, + })); +} + +async function main() { + console.log(`Running ${SCENARIOS.length} scenarios against both classifiers...\n`); + + let textCorrect = 0, toolCorrect = 0, total = 0; + let textParseFailures = 0, toolParseFailures = 0; + let textLatencyTotal = 0, toolLatencyTotal = 0; + + for (const scenario of SCENARIOS) { + console.log(`## ${scenario.name}`); + + // Sequential, not Promise.all: this Ollama server has OLLAMA_NUM_PARALLEL=1, + // so concurrent requests queue behind each other on the server anyway — + // running them "in parallel" here would just make one eat into the + // other's client-side timeout while it waits for a free slot. + const textResult = await runMethod(classifyRelationshipsTextFormat, scenario); + const toolResult = await runMethod(classifyRelationshipsToolCall, scenario); + + textLatencyTotal += textResult.latencyMs; + toolLatencyTotal += toolResult.latencyMs; + if (textResult.guesses.length === 0 && scenario.expected.length > 0) textParseFailures++; + if (toolResult.guesses.length === 0 && scenario.expected.length > 0) toolParseFailures++; + + const textGrades = grade(scenario, textResult.guesses); + const toolGrades = grade(scenario, toolResult.guesses); + + for (let i = 0; i < scenario.expected.length; i++) { + total++; + const t = textGrades[i]!; + const m = toolGrades[i]!; + if (t.correct) textCorrect++; + if (m.correct) toolCorrect++; + + console.log( + ` [${i}] expected=${t.expected.padEnd(11)} text-format=${String(t.got).padEnd(11)}${t.correct ? " ok " : " MISS"} tool-call=${String(m.got).padEnd(11)}${m.correct ? " ok " : " MISS"}` + ); + } + console.log( + ` latency: text-format=${textResult.latencyMs.toFixed(0)}ms tool-call=${toolResult.latencyMs.toFixed(0)}ms` + ); + if (textResult.error) console.log(` text-format error: ${textResult.error}`); + if (toolResult.error) console.log(` tool-call error: ${toolResult.error}`); + console.log(); + } + + console.log("=".repeat(60)); + console.log("SUMMARY"); + console.log("=".repeat(60)); + console.log(`Judgments graded: ${total}`); + console.log(`text-format accuracy: ${textCorrect}/${total} (${((textCorrect / total) * 100).toFixed(1)}%)`); + console.log(`tool-call accuracy: ${toolCorrect}/${total} (${((toolCorrect / total) * 100).toFixed(1)}%)`); + console.log(`text-format parse fails: ${textParseFailures}/${SCENARIOS.length} scenarios (zero guesses returned)`); + console.log(`tool-call parse fails: ${toolParseFailures}/${SCENARIOS.length} scenarios (zero guesses returned)`); + console.log(`text-format avg latency: ${(textLatencyTotal / SCENARIOS.length).toFixed(0)}ms`); + console.log(`tool-call avg latency: ${(toolLatencyTotal / SCENARIOS.length).toFixed(0)}ms`); +} + +await main(); diff --git a/test/forget.test.ts b/test/forget.test.ts new file mode 100644 index 0000000..6c38302 --- /dev/null +++ b/test/forget.test.ts @@ -0,0 +1,198 @@ +/** + * test/forget.test.ts - Tests for the smriti forget (session deletion) layer + * + * Mirrors test/learn-consolidate.test.ts's style: initSmriti(":memory:") so + * QMD's full schema (documents, content_vectors, vectors_vec) exists, a + * seedSession() helper for real message rows, and closeDb() teardown. + */ + +import { test, expect, beforeAll, afterAll } from "bun:test"; +import type { Database } from "bun:sqlite"; +import { + initSmriti, + closeDb, + upsertProject, + upsertSessionMeta, + insertKnowledgeUnit, + promoteKnowledgeUnit, + listKnowledgeUnits, + forgetSession, +} from "../src/db"; +import { listSessions } from "../src/qmd"; +import type { KnowledgeUnit } from "../src/team/types"; + +let db: Database; + +beforeAll(async () => { + db = await initSmriti(":memory:"); +}); + +afterAll(async () => { + await closeDb(); +}); + +function seedSession( + sessionId: string, + projectId: string, + messages: Array<{ role: string; content: string }> +) { + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO memory_sessions (id, title, created_at, updated_at) VALUES (?, ?, ?, ?)` + ).run(sessionId, `Session ${sessionId}`, now, now); + + const insertMsg = db.prepare( + `INSERT INTO memory_messages (session_id, role, content, hash, created_at) VALUES (?, ?, ?, ?, ?)` + ); + for (const [i, m] of messages.entries()) { + insertMsg.run(sessionId, m.role, m.content, `${sessionId}-h${i}`, now); + } + + upsertProject(db, projectId); + upsertSessionMeta(db, sessionId, "claude-code", projectId); +} + +const SAMPLE_MESSAGES = [ + { role: "user", content: "What's our rate limiting strategy?" }, + { role: "assistant", content: "Token bucket, 100 req/min per API key." }, +]; + +// ============================================================================= +// Soft delete +// ============================================================================= + +test("forgetSession soft-deletes by default: hidden from list, kept with includeInactive", () => { + seedSession("soft-s1", "forgetproj", SAMPLE_MESSAGES); + + const result = forgetSession(db, "soft-s1"); + expect(result.hard).toBe(false); + + const active = listSessions(db as any, { includeInactive: false }); + expect(active.map((s: any) => s.id)).not.toContain("soft-s1"); + + const all = listSessions(db as any, { includeInactive: true }); + expect(all.map((s: any) => s.id)).toContain("soft-s1"); + + // Messages are untouched by a soft delete. + const msgs = db + .prepare(`SELECT COUNT(*) as c FROM memory_messages WHERE session_id = ?`) + .get("soft-s1") as { c: number }; + expect(msgs.c).toBe(SAMPLE_MESSAGES.length); +}); + +// ============================================================================= +// Hard delete +// ============================================================================= + +test("forgetSession --hard removes messages, sidecar rows, and unpromoted knowledge units", () => { + seedSession("hard-s1", "forgetproj", SAMPLE_MESSAGES); + + db.prepare( + `INSERT INTO smriti_session_tags (session_id, category_id, confidence, source) VALUES (?, ?, ?, ?)` + ).run("hard-s1", "bug/fix", 0.9, "auto"); + + const segmented: KnowledgeUnit = { + id: "hard-unit-segmented", + topic: "Never promoted", + category: "code/pattern", + relevance: 2, + entities: [], + files: [], + plainText: "Low relevance, never promoted.", + lineRanges: [{ start: 0, end: 1 }], + }; + insertKnowledgeUnit(db, segmented, "hard-s1", "forgetproj", "hard-hash-1"); + db.prepare( + `INSERT INTO smriti_relationships (subject_type, subject_id, predicate, object_type, object_id) VALUES ('knowledge_unit', ?, 'mentions', 'entity', 'rate-limiting')` + ).run("hard-unit-segmented"); + + const result = forgetSession(db, "hard-s1", { hard: true }); + expect(result.hard).toBe(true); + expect(result.unitsDeleted).toBe(1); + + const msgs = db + .prepare(`SELECT COUNT(*) as c FROM memory_messages WHERE session_id = ?`) + .get("hard-s1") as { c: number }; + expect(msgs.c).toBe(0); + + const sessionRow = db.prepare(`SELECT 1 FROM memory_sessions WHERE id = ?`).get("hard-s1"); + expect(sessionRow).toBeNull(); + + const tags = db + .prepare(`SELECT COUNT(*) as c FROM smriti_session_tags WHERE session_id = ?`) + .get("hard-s1") as { c: number }; + expect(tags.c).toBe(0); + + const meta = db.prepare(`SELECT 1 FROM smriti_session_meta WHERE session_id = ?`).get("hard-s1"); + expect(meta).toBeNull(); + + const unit = db.prepare(`SELECT 1 FROM smriti_knowledge_units WHERE id = ?`).get("hard-unit-segmented"); + expect(unit).toBeNull(); + + const edges = db + .prepare(`SELECT COUNT(*) as c FROM smriti_relationships WHERE subject_id = ?`) + .get("hard-unit-segmented") as { c: number }; + expect(edges.c).toBe(0); +}); + +test("forgetSession --hard keeps canonical units and their doc/share unless --purge-shared", () => { + seedSession("hard-s2", "forgetproj", SAMPLE_MESSAGES); + + const canonical: KnowledgeUnit = { + id: "hard-unit-canonical", + topic: "Already shared decision", + category: "architecture/decision", + relevance: 9, + entities: [], + files: [], + plainText: "Already promoted and shared.", + lineRanges: [{ start: 0, end: 1 }], + }; + insertKnowledgeUnit(db, canonical, "hard-s2", "forgetproj", "hard-hash-2"); + promoteKnowledgeUnit(db, "hard-unit-canonical", "knowledge/architecture-decision/doc.md", "share-1"); + db.prepare( + `INSERT INTO smriti_shares (id, session_id, unit_id) VALUES (?, ?, ?)` + ).run("share-1", "hard-s2", "hard-unit-canonical"); + + const result = forgetSession(db, "hard-s2", { hard: true }); + expect(result.canonicalKept).toBe(1); + expect(result.unitsPurged).toBe(0); + + const kept = listKnowledgeUnits(db, { tier: "canonical" }).find( + (u) => u.id === "hard-unit-canonical" + ); + expect(kept).toBeDefined(); + + const share = db.prepare(`SELECT 1 FROM smriti_shares WHERE id = ?`).get("share-1"); + expect(share).toBeTruthy(); +}); + +test("forgetSession --hard --purge-shared removes canonical units and their share row", () => { + seedSession("hard-s3", "forgetproj", SAMPLE_MESSAGES); + + const canonical: KnowledgeUnit = { + id: "purge-unit-canonical", + topic: "Purge me too", + category: "architecture/decision", + relevance: 9, + entities: [], + files: [], + plainText: "Promoted, but this session is being fully purged.", + lineRanges: [{ start: 0, end: 1 }], + }; + insertKnowledgeUnit(db, canonical, "hard-s3", "forgetproj", "hard-hash-3"); + promoteKnowledgeUnit(db, "purge-unit-canonical", "knowledge/architecture-decision/purge-me.md", "share-2"); + db.prepare( + `INSERT INTO smriti_shares (id, session_id, unit_id) VALUES (?, ?, ?)` + ).run("share-2", "hard-s3", "purge-unit-canonical"); + + const result = forgetSession(db, "hard-s3", { hard: true, purgeShared: true }); + expect(result.unitsPurged).toBe(1); + expect(result.canonicalKept).toBe(0); + + const unit = db.prepare(`SELECT 1 FROM smriti_knowledge_units WHERE id = ?`).get("purge-unit-canonical"); + expect(unit).toBeNull(); + + const share = db.prepare(`SELECT 1 FROM smriti_shares WHERE id = ?`).get("share-2"); + expect(share).toBeNull(); +}); diff --git a/test/learn-consolidate.test.ts b/test/learn-consolidate.test.ts new file mode 100644 index 0000000..fa1a5bf --- /dev/null +++ b/test/learn-consolidate.test.ts @@ -0,0 +1,551 @@ +/** + * test/learn-consolidate.test.ts - Tests for continuous knowledge consolidation + * + * Mirrors test/team-segmented.test.ts's style: initSmriti(":memory:"), a + * mocked global.fetch standing in for Ollama, and a scratch tmpDir for + * filesystem output (never process.cwd() — consolidateKnowledge writes real + * files). + */ + +import { test, expect, beforeAll, afterAll, mock } from "bun:test"; +import type { Database } from "bun:sqlite"; +import { mkdirSync, rmSync, writeFileSync, existsSync, readFileSync } from "fs"; +import { join } from "path"; +import { tmpdir } from "os"; +import { + initSmriti, + closeDb, + upsertSessionMeta, + upsertProject, + updateDensityScore, + insertKnowledgeUnit, + listKnowledgeUnits, + promoteKnowledgeUnit, +} from "../src/db"; +import { + consolidateKnowledge, + pruneKnowledge, + classifyRelationshipsTextFormat, + classifyRelationshipsToolCall, +} from "../src/learn/consolidate"; +import { insertRelationship, getRelationships } from "../src/learn/entities"; +import { recall } from "../src/search/recall"; +import type { KnowledgeUnit } from "../src/team/types"; + +// ============================================================================= +// Setup +// ============================================================================= + +let db: Database; +let tmpDir: string; + +beforeAll(async () => { + db = await initSmriti(":memory:"); + tmpDir = join(tmpdir(), `smriti-consolidate-test-${Date.now()}`); + mkdirSync(tmpDir, { recursive: true }); +}); + +afterAll(async () => { + await closeDb(); + try { rmSync(tmpDir, { recursive: true }); } catch {} +}); + +/** Insert a session with real message rows so getSessionMessages() finds content. */ +function seedSession(sessionId: string, projectId: string, messages: Array<{ role: string; content: string }>) { + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO memory_sessions (id, title, created_at, updated_at) VALUES (?, ?, ?, ?)` + ).run(sessionId, `Session ${sessionId}`, now, now); + + const insertMsg = db.prepare( + `INSERT INTO memory_messages (session_id, role, content, hash, created_at) VALUES (?, ?, ?, ?, ?)` + ); + for (const [i, m] of messages.entries()) { + insertMsg.run(sessionId, m.role, m.content, `${sessionId}-h${i}`, now); + } + + upsertProject(db, projectId); + upsertSessionMeta(db, sessionId, "claude-code", projectId); +} + +const DENSE_CONVERSATION = [ + { role: "user", content: "I'm getting a JWT token expiry issue. Sessions timeout after 1 hour but tests expect 24 hours." }, + { role: "assistant", content: "Let me look at the auth middleware to understand the token expiry logic." }, + { role: "user", content: "Found it — src/auth.ts hardcodes 3600 seconds instead of reading JWT_TTL from the environment." }, + { role: "assistant", content: "Updated it to use process.env.JWT_TTL || 3600. Tests pass now." }, +]; + +/** Mock fetch that distinguishes Stage 1 (segmentation) vs Stage 2 (document) calls by prompt content. */ +function mockOllamaFetch(stage1Response: () => object) { + return mock(async (_url: string, init: any) => { + const body = JSON.parse(init.body); + const isStage1 = (body.prompt as string).includes("Knowledge Unit Segmentation"); + if (isStage1) { + return new Response( + JSON.stringify({ response: "```json\n" + JSON.stringify(stage1Response()) + "\n```" }), + { status: 200 } + ); + } + return new Response( + JSON.stringify({ response: "# Consolidated Doc\n\nPolished content." }), + { status: 200 } + ); + }); +} + +// ============================================================================= +// Segment-phase dedup +// ============================================================================= + +test("consolidate dedups knowledge units with identical content across sessions", async () => { + seedSession("dedup-s1", "dedupproj", DENSE_CONVERSATION); + seedSession("dedup-s2", "dedupproj", DENSE_CONVERSATION); + updateDensityScore(db, "dedup-s1", 0.9); + updateDensityScore(db, "dedup-s2", 0.9); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mockOllamaFetch(() => ({ + units: [{ topic: "JWT token expiry bug", category: "bug/fix", relevance: 9, entities: ["JWT"] }], + })) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 0.5, + outputDir: join(tmpDir, "dedup-output"), + }); + + expect(result.sessionsSegmented).toBe(2); + expect(result.unitsStored).toBe(1); + expect(result.unitsSkipped).toBe(1); + expect(result.errors).toEqual([]); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// Promotion threshold +// ============================================================================= + +test("promote phase only promotes units clearing the retrieval/relevance bar", async () => { + const belowThreshold: KnowledgeUnit = { + id: "unit-below", + topic: "Minor formatting note", + category: "code/pattern", + relevance: 3, + entities: [], + files: [], + plainText: "Use consistent indentation.", + lineRanges: [{ start: 0, end: 1 }], + }; + const aboveThreshold: KnowledgeUnit = { + id: "unit-above", + topic: "Redis caching decision", + category: "architecture/decision", + relevance: 9, + entities: ["Redis"], + files: [], + plainText: "Use Redis with a 5-minute TTL for API responses.", + lineRanges: [{ start: 0, end: 1 }], + }; + + insertKnowledgeUnit(db, belowThreshold, "promote-s1", "promoteproj", "hash-below"); + insertKnowledgeUnit(db, aboveThreshold, "promote-s2", "promoteproj", "hash-above"); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mockOllamaFetch(() => ({ units: [] })) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 999, // no sessions qualify for the segment phase — isolates promote phase + minRetrievals: 3, + minRelevance: 8, + outputDir: join(tmpDir, "promote-output"), + }); + + expect(result.unitsPromoted).toBe(1); + expect(result.errors).toEqual([]); + + const canonical = listKnowledgeUnits(db, { tier: "canonical" }); + expect(canonical.map((u) => u.id)).toContain("unit-above"); + expect(canonical.map((u) => u.id)).not.toContain("unit-below"); + + const promoted = canonical.find((u) => u.id === "unit-above")!; + expect(promoted.canonical_doc_path).toContain("architecture-decision"); + expect(promoted.share_id).toBeTruthy(); + + const shareRow = db + .prepare(`SELECT * FROM smriti_shares WHERE unit_id = ?`) + .get("unit-above") as any; + expect(shareRow).toBeTruthy(); + expect(shareRow.session_id).toBe("promote-s2"); + + const writtenFile = join(tmpDir, "promote-output", promoted.canonical_doc_path!); + expect(existsSync(writtenFile)).toBe(true); + + const stillSegmented = listKnowledgeUnits(db, { tier: "segmented" }); + expect(stillSegmented.map((u) => u.id)).toContain("unit-below"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// Graceful degradation +// ============================================================================= + +test("promote phase continues past a per-unit failure and records the error", async () => { + // Pre-create a *file* at the exact path the loop will try to mkdir for + // category "bug/fix" (slug "bug-fix"), forcing that unit's write to throw. + const outputDir = join(tmpDir, "degrade-output"); + const knowledgeDir = join(outputDir, "knowledge"); + mkdirSync(knowledgeDir, { recursive: true }); + writeFileSync(join(knowledgeDir, "bug-fix"), "occupied"); + + const willFail: KnowledgeUnit = { + id: "unit-fails", + topic: "Broken unit", + category: "bug/fix", + relevance: 9, + entities: [], + files: [], + plainText: "This unit's category dir collides with a file.", + lineRanges: [{ start: 0, end: 1 }], + }; + const willSucceed: KnowledgeUnit = { + id: "unit-succeeds", + topic: "Working unit", + category: "topic/learning", + relevance: 9, + entities: [], + files: [], + plainText: "This unit writes fine.", + lineRanges: [{ start: 0, end: 1 }], + }; + + insertKnowledgeUnit(db, willFail, "degrade-s1", "degradeproj", "hash-fails"); + insertKnowledgeUnit(db, willSucceed, "degrade-s2", "degradeproj", "hash-succeeds"); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mockOllamaFetch(() => ({ units: [] })) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 999, + minRetrievals: 999, // exclude leftover segmented units from earlier tests; only relevance-9 units below qualify + minRelevance: 8, + outputDir, + }); + + expect(result.unitsPromoted).toBe(1); + expect(result.errors.length).toBe(1); + expect(result.errors[0]).toContain("unit-fails"); + + const canonical = listKnowledgeUnits(db, { tier: "canonical" }); + expect(canonical.map((u) => u.id)).toContain("unit-succeeds"); + expect(canonical.map((u) => u.id)).not.toContain("unit-fails"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// Retrieval tracking (recall -> incrementRetrievalCount) +// ============================================================================= + +test("recall increments retrieval_count for knowledge units of the recalled session", async () => { + seedSession("track-s1", "trackproj", [ + { role: "user", content: "How do we configure the rate limiter for the public API?" }, + { role: "assistant", content: "Use a token bucket with 100 requests per minute per API key." }, + ]); + + const unit: KnowledgeUnit = { + id: "unit-tracked", + topic: "Rate limiter config", + category: "code/pattern", + relevance: 7, + entities: [], + files: [], + plainText: "Token bucket rate limiting for the public API.", + lineRanges: [{ start: 0, end: 1 }], + }; + insertKnowledgeUnit(db, unit, "track-s1", "trackproj", "hash-tracked"); + + await recall(db, "rate limiter", { project: "trackproj" }); + + const tracked = db + .prepare(`SELECT retrieval_count FROM smriti_knowledge_units WHERE id = ?`) + .get("unit-tracked") as { retrieval_count: number }; + expect(tracked).toBeDefined(); + expect(tracked.retrieval_count).toBeGreaterThanOrEqual(1); +}); + +test("recall does not throw for sessions with no consolidated knowledge units", async () => { + seedSession("untracked-s1", "untrackedproj", [ + { role: "user", content: "What's our deploy process look like?" }, + { role: "assistant", content: "Push to main triggers the CI pipeline and auto-deploys." }, + ]); + + await expect(recall(db, "deploy process", { project: "untrackedproj" })).resolves.toBeDefined(); +}); + +// ============================================================================= +// Relationship classification (text-format vs tool-call) +// ============================================================================= + +const RELATION_UNIT = { topic: "Post-filtering for vector search", category: "architecture/decision", plainText: "Switched to post-filtering." }; +const RELATION_CANDIDATES = [ + { id: "cand-0", topic: "Pre-filtering for vector search", category: "architecture/decision", plain_text: "Decided to pre-filter." }, + { id: "cand-1", topic: "Unrelated CSS fix", category: "bug/fix", plain_text: "Fixed a hover overlay." }, +]; + +test("classifyRelationshipsTextFormat parses RELATION lines even without literal brackets", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async () => + new Response(JSON.stringify({ response: "RELATION 0: supersedes\nRELATION [1]: none" }), { status: 200 }) + ) as any; + + try { + const guesses = await classifyRelationshipsTextFormat(RELATION_UNIT, RELATION_CANDIDATES, "test-model"); + expect(guesses).toEqual([ + { index: 0, predicate: "supersedes" }, + { index: 1, predicate: "none" }, + ]); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("classifyRelationshipsTextFormat returns nothing parseable when the model drifts off-format", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async () => + new Response(JSON.stringify({ response: "I think candidate 0 is superseded by the new unit." }), { status: 200 }) + ) as any; + + try { + const guesses = await classifyRelationshipsTextFormat(RELATION_UNIT, RELATION_CANDIDATES, "test-model"); + expect(guesses).toEqual([]); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("classifyRelationshipsToolCall parses structured tool_calls output", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async () => + new Response( + JSON.stringify({ + model: "test-model", + message: { + role: "assistant", + content: "", + tool_calls: [ + { + function: { + name: "record_relationships", + arguments: { + relationships: [ + { index: 0, predicate: "supersedes" }, + { index: 1, predicate: "none" }, + ], + }, + }, + }, + ], + }, + done: true, + }), + { status: 200 } + ) + ) as any; + + try { + const guesses = await classifyRelationshipsToolCall(RELATION_UNIT, RELATION_CANDIDATES, "test-model"); + expect(guesses).toEqual([ + { index: 0, predicate: "supersedes" }, + { index: 1, predicate: "none" }, + ]); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("classifyRelationshipsToolCall drops entries with out-of-range indices or invalid predicates", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async () => + new Response( + JSON.stringify({ + model: "test-model", + message: { + role: "assistant", + content: "", + tool_calls: [ + { + function: { + name: "record_relationships", + arguments: { + relationships: [ + { index: 0, predicate: "supersedes" }, + { index: 99, predicate: "relatesTo" }, + { index: 1, predicate: "maybe" }, + ], + }, + }, + }, + ], + }, + done: true, + }), + { status: 200 } + ) + ) as any; + + try { + const guesses = await classifyRelationshipsToolCall(RELATION_UNIT, RELATION_CANDIDATES, "test-model"); + expect(guesses).toEqual([{ index: 0, predicate: "supersedes" }]); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("classifyRelationshipsToolCall returns nothing when the model answers without calling the tool", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async () => + new Response( + JSON.stringify({ + model: "test-model", + message: { role: "assistant", content: "Candidate 0 looks superseded." }, + done: true, + }), + { status: 200 } + ) + ) as any; + + try { + const guesses = await classifyRelationshipsToolCall(RELATION_UNIT, RELATION_CANDIDATES, "test-model"); + expect(guesses).toEqual([]); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// Prune (stale segmented units, superseded canonical units) +// ============================================================================= + +function backdateUnit(unitId: string, daysAgo: number) { + db.prepare( + `UPDATE smriti_knowledge_units SET created_at = datetime('now', '-' || ? || ' days') WHERE id = ?` + ).run(daysAgo, unitId); +} + +test("pruneKnowledge dry-run reports stale segmented units without deleting them", async () => { + const stale: KnowledgeUnit = { + id: "prune-stale-1", + topic: "Never promoted, never retrieved", + category: "code/pattern", + relevance: 3, + entities: [], + files: [], + plainText: "Low relevance, sat unused for weeks.", + lineRanges: [{ start: 0, end: 1 }], + }; + insertKnowledgeUnit(db, stale, "prune-s1", "pruneproj", "prune-hash-1"); + backdateUnit("prune-stale-1", 45); + + const result = await pruneKnowledge(db, { dryRun: true, pruneStaleDays: 30, minRelevance: 8 }); + + expect(result.unitsPruned).toBe(0); + expect(result.unitsArchived).toBe(0); + expect(result.pruneCandidates?.map((c) => c.id)).toContain("prune-stale-1"); + + const stillThere = listKnowledgeUnits(db, { tier: "segmented" }).find((u) => u.id === "prune-stale-1"); + expect(stillThere).toBeDefined(); +}); + +test("pruneKnowledge --apply deletes stale segmented units and their relationship edges", async () => { + const stale: KnowledgeUnit = { + id: "prune-apply-1", + topic: "Deletable stale unit", + category: "code/pattern", + relevance: 2, + entities: [], + files: [], + plainText: "Stale and about to be deleted.", + lineRanges: [{ start: 0, end: 1 }], + }; + insertKnowledgeUnit(db, stale, "prune-s2", "pruneproj", "prune-hash-2"); + backdateUnit("prune-apply-1", 45); + insertRelationship(db, "knowledge_unit", "prune-apply-1", "mentions", "entity", "some-entity"); + + const result = await pruneKnowledge(db, { dryRun: false, pruneStaleDays: 30, minRelevance: 8 }); + + expect(result.unitsPruned).toBeGreaterThanOrEqual(1); + const deleted = db.prepare(`SELECT 1 FROM smriti_knowledge_units WHERE id = ?`).get("prune-apply-1"); + expect(deleted).toBeNull(); + + const edges = getRelationships(db, { subjectType: "knowledge_unit", subjectId: "prune-apply-1" }); + expect(edges.length).toBe(0); +}); + +test("pruneKnowledge never prunes high-relevance segmented units even at zero retrievals", async () => { + const highRelevance: KnowledgeUnit = { + id: "prune-high-relevance", + topic: "One consolidate run away from promoting", + category: "architecture/decision", + relevance: 9, + entities: [], + files: [], + plainText: "High relevance, just hasn't been promoted yet.", + lineRanges: [{ start: 0, end: 1 }], + }; + insertKnowledgeUnit(db, highRelevance, "prune-s3", "pruneproj", "prune-hash-3"); + backdateUnit("prune-high-relevance", 45); + + const result = await pruneKnowledge(db, { dryRun: true, pruneStaleDays: 30, minRelevance: 8 }); + + expect(result.pruneCandidates?.map((c) => c.id)).not.toContain("prune-high-relevance"); +}); + +test("pruneKnowledge archives superseded canonical units and appends a banner to their doc", async () => { + const outputDir = join(tmpDir, "prune-archive-output"); + const docRelPath = "knowledge/architecture-decision/old-doc.md"; + const docFullPath = join(outputDir, docRelPath); + mkdirSync(join(outputDir, "knowledge/architecture-decision"), { recursive: true }); + writeFileSync(docFullPath, "---\nid: old-unit\n---\n\n# Old guidance\n\nUse a 1-minute TTL."); + + const oldUnit: KnowledgeUnit = { + id: "prune-superseded", topic: "Old Redis TTL guidance", category: "architecture/decision", relevance: 9, + entities: [], files: [], plainText: "Use a 1-minute TTL.", lineRanges: [{ start: 0, end: 1 }], + }; + const newUnit: KnowledgeUnit = { + id: "prune-superseder", topic: "New Redis TTL guidance", category: "architecture/decision", relevance: 9, + entities: [], files: [], plainText: "Use a 5-minute TTL instead.", lineRanges: [{ start: 0, end: 1 }], + }; + insertKnowledgeUnit(db, oldUnit, "prune-s4", "pruneproj", "prune-hash-old"); + insertKnowledgeUnit(db, newUnit, "prune-s5", "pruneproj", "prune-hash-new"); + promoteKnowledgeUnit(db, "prune-superseded", docRelPath, "prune-share-1"); + promoteKnowledgeUnit(db, "prune-superseder", "knowledge/architecture-decision/new-doc.md", "prune-share-2"); + insertRelationship(db, "knowledge_unit", "prune-superseder", "supersedes", "knowledge_unit", "prune-superseded", { + source: "llm", + }); + + const result = await pruneKnowledge(db, { dryRun: false, outputDir }); + + expect(result.unitsArchived).toBe(1); + + const archived = listKnowledgeUnits(db, { tier: "archived" }).find((u) => u.id === "prune-superseded"); + expect(archived).toBeDefined(); + expect(archived!.archived_reason).toBe("superseded"); + + const docContent = readFileSync(docFullPath, "utf-8"); + expect(docContent).toContain("Archived"); + expect(docContent).toContain("New Redis TTL guidance"); + + // The supersedes edge that justified archiving (and any mentions edges) + // are the audit trail — left untouched, not cascade-deleted. + const supersedeEdge = getRelationships(db, { + subjectType: "knowledge_unit", subjectId: "prune-superseder", predicate: "supersedes", objectId: "prune-superseded", + }); + expect(supersedeEdge.length).toBe(1); +}); diff --git a/test/learn-entities-sync.test.ts b/test/learn-entities-sync.test.ts new file mode 100644 index 0000000..8b850c4 --- /dev/null +++ b/test/learn-entities-sync.test.ts @@ -0,0 +1,166 @@ +/** + * test/learn-entities-sync.test.ts - Team/org propagation of the entities + + * relationships layer via the existing share/sync git round-trip. + * + * This is the test that directly answers "how does this reach the org/team + * level": two independent in-memory DBs stand in for two teammates' machines, + * connected only through a shared tmp `.smriti/` directory (config.json + + * knowledge/*.md), exactly like real git-committed team knowledge. + */ + +import { test, expect, beforeAll, afterAll, mock } from "bun:test"; +import type { Database } from "bun:sqlite"; +import { mkdirSync, rmSync } from "fs"; +import { join } from "path"; +import { tmpdir } from "os"; +import { initSmriti, closeDb, insertKnowledgeUnit, upsertProject } from "../src/db"; +import { readConfig, writeConfig, exportEntities } from "../src/team/config"; +import { syncTeamKnowledge } from "../src/team/sync"; +import { consolidateKnowledge } from "../src/learn/consolidate"; +import { getEntity, resolveEntity, insertRelationship, getRelationships } from "../src/learn/entities"; +import type { KnowledgeUnit } from "../src/team/types"; + +// Two independent :memory: databases (via separate initSmriti calls) stand in +// for two teammates' machines. initSmriti's underlying store singleton is +// process-wide, but every call site below threads the returned `db` handle +// explicitly rather than going through the singleton getDb(), so the two +// stay genuinely isolated for everything this test touches. +let dbA: Database; +let dbB: Database; +let sharedDir: string; + +beforeAll(async () => { + dbA = await initSmriti(":memory:"); + dbB = await initSmriti(":memory:"); + sharedDir = join(tmpdir(), `smriti-propagation-test-${Date.now()}`); + mkdirSync(sharedDir, { recursive: true }); + + // Pre-existing FK requirement of syncTeamKnowledge/upsertSessionMeta, + // unrelated to the entities/relationships work: the importing machine + // must already have the project registered locally (smriti_session_meta + // FKs to smriti_projects). Both "teammates" in this test are in the same project. + upsertProject(dbA, "propproj"); + upsertProject(dbB, "propproj"); +}); + +afterAll(async () => { + await closeDb(); + try { rmSync(sharedDir, { recursive: true }); } catch {} +}); + +test("entity canonical id and unit-relationship edges converge across two machines via share -> sync", async () => { + // --- Machine A: create + promote a unit that mentions "Redis" --- + const unit: KnowledgeUnit = { + id: "prop-unit-a", + topic: "Redis TTL guidance", + category: "architecture/decision", + relevance: 9, + entities: ["Redis"], + files: [], + plainText: "Use a 5-minute TTL for API response caching.", + lineRanges: [], + }; + insertKnowledgeUnit(dbA, unit, "prop-session-a", "propproj", "prop-hash-a"); + const redisIdOnA = resolveEntity(dbA, "Redis")!; + insertRelationship(dbA, "knowledge_unit", "prop-unit-a", "mentions", "entity", redisIdOnA); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async () => + new Response(JSON.stringify({ response: "# Redis TTL Guidance\n\nUse a 5-minute TTL." }), { status: 200 }) + ) as any; + + try { + const result = await consolidateKnowledge(dbA, { + minDensity: 999, // no segment-phase sessions on A — isolates the promote phase + minRetrievals: 999, + minRelevance: 8, // unit's relevance (9) qualifies + outputDir: sharedDir, + }); + expect(result.unitsPromoted).toBe(1); + expect(result.errors).toEqual([]); + } finally { + globalThis.fetch = originalFetch; + } + + // --- Machine A "shares": publish its canonical entity registry to the shared config.json --- + // (this is the exact operation writeManifest performs in src/team/share.ts, just called + // directly here to isolate propagation from the rest of the share pipeline's session-querying) + const existingConfig = readConfig(sharedDir); + await writeConfig(sharedDir, { ...existingConfig, version: 2, entities: exportEntities(dbA) }); + + // --- Machine B has never seen "Redis" before --- + expect(getEntity(dbB, "redis")).toBeNull(); + + const syncResult = await syncTeamKnowledge(dbB, { inputDir: sharedDir }); + + expect(syncResult.errors).toEqual([]); + expect(syncResult.entitiesImported).toBeGreaterThanOrEqual(1); + expect(syncResult.imported).toBeGreaterThanOrEqual(1); + + // The entity converged onto the SAME canonical id — not an independently re-slugified duplicate. + const redisOnB = getEntity(dbB, "redis"); + expect(redisOnB).toBeTruthy(); + expect(redisOnB!.id).toBe(redisIdOnA); + expect(redisOnB!.aliases).toContain("Redis"); + + // The unit's "mentions" edge was re-created on B, referencing that same entity id. + const bMentions = getRelationships(dbB, { + subjectType: "knowledge_unit", + subjectId: "prop-unit-a", + predicate: "mentions", + objectType: "entity", + }); + expect(bMentions.map((r) => r.object_id)).toContain(redisIdOnA); +}); + +test("supersedes edges between shared units survive the round-trip with no canonicalization needed", async () => { + // Machine A: an existing unit, and a new one that supersedes it — both promoted. + const existing: KnowledgeUnit = { + id: "prop-superseded-unit", topic: "Old Redis TTL", category: "architecture/decision", relevance: 8, + entities: ["Redis"], files: [], plainText: "Use a 1-minute TTL.", lineRanges: [], + }; + const superseding: KnowledgeUnit = { + id: "prop-superseding-unit", topic: "New Redis TTL", category: "architecture/decision", relevance: 9, + entities: ["Redis"], files: [], plainText: "Use a 5-minute TTL instead.", lineRanges: [], + }; + insertKnowledgeUnit(dbA, existing, "prop-session-b1", "propproj", "prop-hash-existing"); + insertKnowledgeUnit(dbA, superseding, "prop-session-b2", "propproj", "prop-hash-superseding"); + const redisId = resolveEntity(dbA, "Redis")!; + insertRelationship(dbA, "knowledge_unit", "prop-superseded-unit", "mentions", "entity", redisId); + insertRelationship(dbA, "knowledge_unit", "prop-superseding-unit", "mentions", "entity", redisId); + // Simulate what promote-phase LLM relationship inference would have discovered: + insertRelationship(dbA, "knowledge_unit", "prop-superseding-unit", "supersedes", "knowledge_unit", "prop-superseded-unit", { source: "llm" }); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async () => + new Response(JSON.stringify({ response: "# Doc\n\nContent." }), { status: 200 }) + ) as any; + + try { + // Only "prop-superseding-unit" clears the bar this round (existing unit stays at relevance 8 + // < minRelevance 8.5, so we promote exactly the one whose frontmatter should carry the edge). + const result = await consolidateKnowledge(dbA, { + minDensity: 999, + minRetrievals: 999, + minRelevance: 8.5, + outputDir: sharedDir, + }); + expect(result.unitsPromoted).toBe(1); + } finally { + globalThis.fetch = originalFetch; + } + + await writeConfig(sharedDir, { ...readConfig(sharedDir), version: 2, entities: exportEntities(dbA) }); + + const syncResult = await syncTeamKnowledge(dbB, { inputDir: sharedDir }); + expect(syncResult.errors).toEqual([]); + + const bEdges = getRelationships(dbB, { + subjectType: "knowledge_unit", + subjectId: "prop-superseding-unit", + predicate: "supersedes", + }); + // No entity-style canonicalization needed here: unit ids are portable UUIDs, + // so the edge lands referencing the exact same object id on both machines. + expect(bEdges.map((r) => r.object_id)).toContain("prop-superseded-unit"); +}); diff --git a/test/learn-entities.test.ts b/test/learn-entities.test.ts new file mode 100644 index 0000000..90e65f8 --- /dev/null +++ b/test/learn-entities.test.ts @@ -0,0 +1,425 @@ +/** + * test/learn-entities.test.ts - Tests for canonical entity resolution and + * relationship triples (RDF-inspired knowledge graph layer). + * + * Mirrors test/learn-consolidate.test.ts's style: initSmriti(":memory:"), + * mocked global.fetch standing in for Ollama, scratch tmpDir for output. + */ + +import { test, expect, beforeAll, afterAll, mock } from "bun:test"; +import type { Database } from "bun:sqlite"; +import { mkdirSync, rmSync } from "fs"; +import { join } from "path"; +import { tmpdir } from "os"; +import { + initSmriti, + closeDb, + upsertSessionMeta, + upsertProject, + updateDensityScore, + insertKnowledgeUnit, + findPromotableUnits, +} from "../src/db"; +import { + resolveEntity, + getEntity, + findEntity, + insertRelationship, + getRelationships, + findRelatedCandidates, + getUnitsForEntity, +} from "../src/learn/entities"; +import { consolidateKnowledge } from "../src/learn/consolidate"; +import type { KnowledgeUnit } from "../src/team/types"; + +let db: Database; +let tmpDir: string; + +beforeAll(async () => { + db = await initSmriti(":memory:"); + tmpDir = join(tmpdir(), `smriti-entities-test-${Date.now()}`); + mkdirSync(tmpDir, { recursive: true }); +}); + +afterAll(async () => { + await closeDb(); + try { rmSync(tmpDir, { recursive: true }); } catch {} +}); + +function seedSession(sessionId: string, projectId: string, messages: Array<{ role: string; content: string }>) { + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO memory_sessions (id, title, created_at, updated_at) VALUES (?, ?, ?, ?)` + ).run(sessionId, `Session ${sessionId}`, now, now); + + const insertMsg = db.prepare( + `INSERT INTO memory_messages (session_id, role, content, hash, created_at) VALUES (?, ?, ?, ?, ?)` + ); + for (const [i, m] of messages.entries()) { + insertMsg.run(sessionId, m.role, m.content, `${sessionId}-h${i}`, now); + } + + upsertProject(db, projectId); + upsertSessionMeta(db, sessionId, "claude-code", projectId); +} + +type RelationGuessLite = { index: number; predicate: string }; + +/** + * Stage 1 (segmentSession) and Stage 2 (generateDocument) go through + * callOllama's /api/generate, with a `prompt` field on the request body. + * Promote-time relationship inference (classifyRelationshipsToolCall) goes + * through ollamaChat's /api/chat instead — no `prompt` field, a `messages` + * array plus a native tool call in the response instead of free text. + */ +function mockOllamaFetch(handlers: { stage1?: () => object; relation?: () => RelationGuessLite[]; stage2?: () => string }) { + return mock(async (_url: string, init: any) => { + const body = JSON.parse(init.body); + + if (typeof body.prompt === "string") { + if (body.prompt.includes("Knowledge Unit Segmentation")) { + return new Response( + JSON.stringify({ response: "```json\n" + JSON.stringify((handlers.stage1 ?? (() => ({ units: [] })))()) + "\n```" }), + { status: 200 } + ); + } + return new Response( + JSON.stringify({ response: (handlers.stage2 ?? (() => "# Doc\n\nContent."))() }), + { status: 200 } + ); + } + + return new Response( + JSON.stringify({ + model: "test-model", + message: { + role: "assistant", + content: "", + tool_calls: [ + { + function: { + name: "record_relationships", + arguments: { relationships: (handlers.relation ?? (() => []))() }, + }, + }, + ], + }, + done: true, + }), + { status: 200 } + ); + }); +} + +// ============================================================================= +// Entity resolution +// ============================================================================= + +test("resolveEntity merges exact-normalize variants (case/whitespace) onto one canonical id", () => { + const id1 = resolveEntity(db, "JWT")!; + const id2 = resolveEntity(db, " jwt ")!; + const id3 = resolveEntity(db, "JWT."); + + expect(id1).toBe(id2); + expect(id1).toBe(id3); + + const entity = getEntity(db, id1)!; + expect(entity.label).toBe("JWT"); // first-seen label wins + expect(entity.aliases).toContain("JWT"); + expect(entity.aliases).toContain("jwt"); + expect(entity.mention_count).toBe(3); +}); + +test("resolveEntity does not merge genuinely different wordings for the same concept", () => { + const jwtId = resolveEntity(db, "distinct-jwt-test")!; + const fullFormId = resolveEntity(db, "distinct-json-web-token-test")!; + + expect(jwtId).not.toBe(fullFormId); +}); + +test("resolveEntity returns null for blank labels", () => { + expect(resolveEntity(db, " ")).toBeNull(); +}); + +test("findEntity looks up by exact id or by label", () => { + resolveEntity(db, "Redis"); + expect(findEntity(db, "Redis")?.id).toBe("redis"); + expect(findEntity(db, "redis")?.id).toBe("redis"); + expect(findEntity(db, "nonexistent-entity-xyz")).toBeNull(); +}); + +// ============================================================================= +// Relationship triples +// ============================================================================= + +test("insertRelationship dedups via the UNIQUE constraint", () => { + insertRelationship(db, "knowledge_unit", "unit-a", "mentions", "entity", "redis"); + insertRelationship(db, "knowledge_unit", "unit-a", "mentions", "entity", "redis"); + + const rows = getRelationships(db, { subjectType: "knowledge_unit", subjectId: "unit-a", predicate: "mentions" }); + expect(rows.length).toBe(1); +}); + +test("getRelationships supports single-triple-pattern lookup", () => { + insertRelationship(db, "knowledge_unit", "unit-b", "supersedes", "knowledge_unit", "unit-a", { source: "llm" }); + + const bySubject = getRelationships(db, { subjectId: "unit-b" }); + expect(bySubject.some((r) => r.predicate === "supersedes" && r.object_id === "unit-a")).toBe(true); + + const byPredicate = getRelationships(db, { predicate: "supersedes" }); + expect(byPredicate.length).toBeGreaterThanOrEqual(1); +}); + +// ============================================================================= +// Segment phase: mentions edges created with no extra LLM calls +// ============================================================================= + +test("consolidate segment phase creates mentions edges for every stored entity, no extra LLM calls", async () => { + seedSession("ent-s1", "entproj", [ + { role: "user", content: "We need to decide on a caching strategy for the API. Considering Redis vs in-memory caching." }, + { role: "assistant", content: "Redis is better — it's external state, handles multi-instance, fast, and proven." }, + ]); + updateDensityScore(db, "ent-s1", 0.9); + + let fetchCallCount = 0; + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async (_url: string, init: any) => { + fetchCallCount++; + const body = JSON.parse(init.body); + const isStage1 = (body.prompt as string).includes("Knowledge Unit Segmentation"); + if (isStage1) { + // relevance kept low (1) deliberately: this unit stays in the shared + // test DB as tier='segmented' after this test, and must never satisfy + // a later test's minRelevance threshold (e.g. 8) via leftover state. + return new Response( + JSON.stringify({ + response: "```json\n" + JSON.stringify({ + units: [{ topic: "Redis caching decision", category: "architecture/decision", relevance: 1, entities: ["Redis", "Caching"] }], + }) + "\n```", + }), + { status: 200 } + ); + } + return new Response(JSON.stringify({ response: "unexpected call" }), { status: 200 }); + }) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 0.5, + minRetrievals: 999, + minRelevance: 999, // nothing promotable — isolates the segment phase + outputDir: join(tmpDir, "ent-output"), + }); + + expect(result.unitsStored).toBe(1); + expect(fetchCallCount).toBe(1); // segmentation only, no promote-time LLM calls + + const unitRow = db + .prepare(`SELECT id FROM smriti_knowledge_units WHERE session_id = 'ent-s1'`) + .get() as { id: string }; + + const mentions = getRelationships(db, { + subjectType: "knowledge_unit", + subjectId: unitRow.id, + predicate: "mentions", + }); + expect(mentions.length).toBe(2); + const objectIds = mentions.map((m) => m.object_id).sort(); + expect(objectIds).toEqual(["caching", "redis"]); + + const unitsForRedis = getUnitsForEntity(db, "redis"); + expect(unitsForRedis.map((u) => u.id)).toContain(unitRow.id); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// minEntityReach promotion criterion +// ============================================================================= + +test("minEntityReach promotes a unit whose entity is shared by >= K other units, even at 0 retrievals", () => { + const shared: KnowledgeUnit = { + id: "reach-unit-1", topic: "Topic A", category: "code/pattern", relevance: 2, + entities: [], files: [], plainText: "content A", lineRanges: [], + }; + const sharedOther1: KnowledgeUnit = { + id: "reach-unit-2", topic: "Topic B", category: "code/pattern", relevance: 2, + entities: [], files: [], plainText: "content B", lineRanges: [], + }; + const sharedOther2: KnowledgeUnit = { + id: "reach-unit-3", topic: "Topic C", category: "code/pattern", relevance: 2, + entities: [], files: [], plainText: "content C", lineRanges: [], + }; + + insertKnowledgeUnit(db, shared, "reach-s1", "reachproj", "reach-hash-1"); + insertKnowledgeUnit(db, sharedOther1, "reach-s2", "reachproj", "reach-hash-2"); + insertKnowledgeUnit(db, sharedOther2, "reach-s3", "reachproj", "reach-hash-3"); + + const entityId = resolveEntity(db, "shared-webhook-retries")!; + insertRelationship(db, "knowledge_unit", "reach-unit-1", "mentions", "entity", entityId); + insertRelationship(db, "knowledge_unit", "reach-unit-2", "mentions", "entity", entityId); + insertRelationship(db, "knowledge_unit", "reach-unit-3", "mentions", "entity", entityId); + + // Below scalar thresholds (relevance=2, retrieval_count=0) but 2 OTHER units share the entity. + const promotableWithReach = findPromotableUnits(db, 999, 999, 2); + expect(promotableWithReach.map((u) => u.id)).toContain("reach-unit-1"); + + // Without minEntityReach, the same unit is not promotable. + const promotableWithoutReach = findPromotableUnits(db, 999, 999); + expect(promotableWithoutReach.map((u) => u.id)).not.toContain("reach-unit-1"); +}); + +// ============================================================================= +// Promote-phase relationship inference (LLM-gated, bounded) +// ============================================================================= + +test("promote phase persists LLM-inferred relatesTo/supersedes/contradicts edges", async () => { + const existing: KnowledgeUnit = { + id: "infer-existing", topic: "Old Redis TTL guidance", category: "architecture/decision", relevance: 5, + entities: [], files: [], plainText: "Use a 1-minute TTL.", lineRanges: [], + }; + insertKnowledgeUnit(db, existing, "infer-s1", "inferproj", "infer-hash-existing"); + const redisId = resolveEntity(db, "infer-redis")!; + insertRelationship(db, "knowledge_unit", "infer-existing", "mentions", "entity", redisId); + + const promoted: KnowledgeUnit = { + id: "infer-new", topic: "New Redis TTL guidance", category: "architecture/decision", relevance: 9, + entities: ["infer-redis"], files: [], plainText: "Use a 5-minute TTL instead.", lineRanges: [], + }; + insertKnowledgeUnit(db, promoted, "infer-s2", "inferproj", "infer-hash-new"); + insertRelationship(db, "knowledge_unit", "infer-new", "mentions", "entity", redisId); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mockOllamaFetch({ relation: () => [{ index: 0, predicate: "supersedes" }] }) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 999, // no segment-phase sessions + minRetrievals: 999, + minRelevance: 8, // only infer-new (relevance 9) qualifies + outputDir: join(tmpDir, "infer-output"), + }); + + expect(result.unitsPromoted).toBe(1); + expect(result.errors).toEqual([]); + + const edges = getRelationships(db, { subjectType: "knowledge_unit", subjectId: "infer-new", predicate: "supersedes" }); + expect(edges.length).toBe(1); + expect(edges[0].object_id).toBe("infer-existing"); + expect(edges[0].source).toBe("llm"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("promote phase never asserts a directional predicate in both directions for the same pair", async () => { + // Two units sharing an entity, BOTH clearing the promotion bar in the same + // run — each independently asks "do I supersede the other" and (with a + // naive LLM/mock that doesn't reason about recency) can get "yes" from both + // sides. Only one direction should end up persisted. + const unitA: KnowledgeUnit = { + id: "bidir-unit-a", topic: "Redis TTL: 1 minute", category: "architecture/decision", relevance: 9, + entities: [], files: [], plainText: "Use a 1-minute TTL.", lineRanges: [], + }; + const unitB: KnowledgeUnit = { + id: "bidir-unit-b", topic: "Redis TTL: 15 minutes", category: "architecture/decision", relevance: 9, + entities: [], files: [], plainText: "Use a 15-minute TTL instead.", lineRanges: [], + }; + insertKnowledgeUnit(db, unitA, "bidir-s1", "bidirproj", "bidir-hash-a"); + insertKnowledgeUnit(db, unitB, "bidir-s2", "bidirproj", "bidir-hash-b"); + const entityId = resolveEntity(db, "bidir-redis")!; + insertRelationship(db, "knowledge_unit", "bidir-unit-a", "mentions", "entity", entityId); + insertRelationship(db, "knowledge_unit", "bidir-unit-b", "mentions", "entity", entityId); + + const originalFetch = globalThis.fetch; + // Always answers "supersedes" regardless of which side is asking — the + // worst case for this bug, and realistic for a small/local model given a + // prompt with no explicit recency signal. + globalThis.fetch = mockOllamaFetch({ relation: () => [{ index: 0, predicate: "supersedes" }] }) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 999, + minRetrievals: 999, + minRelevance: 8, // both unitA and unitB qualify + outputDir: join(tmpDir, "bidir-output"), + }); + + expect(result.unitsPromoted).toBe(2); + + const aToB = getRelationships(db, { + subjectType: "knowledge_unit", subjectId: "bidir-unit-a", predicate: "supersedes", objectId: "bidir-unit-b", + }); + const bToA = getRelationships(db, { + subjectType: "knowledge_unit", subjectId: "bidir-unit-b", predicate: "supersedes", objectId: "bidir-unit-a", + }); + // Exactly one direction persisted, never both. + expect(aToB.length + bToA.length).toBe(1); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("promote phase relationship inference is best-effort: a broken LLM response doesn't block promotion", async () => { + const existing: KnowledgeUnit = { + id: "badllm-existing", topic: "Existing", category: "code/pattern", relevance: 5, + entities: [], files: [], plainText: "content", lineRanges: [], + }; + insertKnowledgeUnit(db, existing, "badllm-s1", "badllmproj", "badllm-hash-existing"); + const entId = resolveEntity(db, "badllm-entity")!; + insertRelationship(db, "knowledge_unit", "badllm-existing", "mentions", "entity", entId); + + const promoted: KnowledgeUnit = { + id: "badllm-new", topic: "New", category: "code/pattern", relevance: 9, + entities: ["badllm-entity"], files: [], plainText: "content", lineRanges: [], + }; + insertKnowledgeUnit(db, promoted, "badllm-s2", "badllmproj", "badllm-hash-new"); + insertRelationship(db, "knowledge_unit", "badllm-new", "mentions", "entity", entId); + + const originalFetch = globalThis.fetch; + globalThis.fetch = mock(async (_url: string, init: any) => { + const body = JSON.parse(init.body); + const prompt = body.prompt as string; + if (prompt.includes("CANDIDATES")) throw new Error("connection refused"); + return new Response(JSON.stringify({ response: "# Doc\n\nContent." }), { status: 200 }); + }) as any; + + try { + const result = await consolidateKnowledge(db, { + minDensity: 999, + minRetrievals: 999, + minRelevance: 8, + outputDir: join(tmpDir, "badllm-output"), + }); + + // Promotion itself succeeds even though relationship inference failed. + expect(result.unitsPromoted).toBe(1); + expect(result.errors).toEqual([]); + + const edges = getRelationships(db, { subjectType: "knowledge_unit", subjectId: "badllm-new" }) + .filter((r) => r.predicate !== "mentions"); + expect(edges.length).toBe(0); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ============================================================================= +// findRelatedCandidates +// ============================================================================= + +test("findRelatedCandidates finds other units sharing a canonical entity, excluding self", () => { + const a: KnowledgeUnit = { id: "cand-a", topic: "A", category: "code/pattern", relevance: 5, entities: [], files: [], plainText: "a", lineRanges: [] }; + const b: KnowledgeUnit = { id: "cand-b", topic: "B", category: "code/pattern", relevance: 5, entities: [], files: [], plainText: "b", lineRanges: [] }; + insertKnowledgeUnit(db, a, "cand-s1", "candproj", "cand-hash-a"); + insertKnowledgeUnit(db, b, "cand-s2", "candproj", "cand-hash-b"); + + const sharedEntity = resolveEntity(db, "cand-shared-entity")!; + insertRelationship(db, "knowledge_unit", "cand-a", "mentions", "entity", sharedEntity); + insertRelationship(db, "knowledge_unit", "cand-b", "mentions", "entity", sharedEntity); + + const candidates = findRelatedCandidates(db, "cand-a", 5); + expect(candidates.map((c) => c.id)).toEqual(["cand-b"]); + expect(candidates.map((c) => c.id)).not.toContain("cand-a"); +}); diff --git a/test/recall-quality.test.ts b/test/recall-quality.test.ts new file mode 100644 index 0000000..99ccfa1 --- /dev/null +++ b/test/recall-quality.test.ts @@ -0,0 +1,38 @@ +/** + * test/recall-quality.test.ts - Tier 1 of the recall-quality harness: the + * BM25-only scenarios from test/eval/fixtures/, run deterministically with + * no embedding backend needed (recall's project-filtered path never touches + * vectors; recallMemories's vector search silently no-ops with none). + * + * This is the CI-safe subset — it catches regressions in recall()'s + * filtering/dedup/RRF/density-blending wiring automatically. The full + * fixture set (including embedding-dependent scenarios) runs manually via + * `bun run eval:recall` (test/eval/recall-quality.eval.ts). + */ + +import { test, expect } from "bun:test"; +import { initSmriti, closeDb } from "../src/db"; +import { CI_SCENARIOS } from "./eval/fixtures/index"; +import { seedScenario } from "./eval/fixtures/seed"; +import { runProbe } from "./eval/fixtures/run"; + +for (const scenario of CI_SCENARIOS) { + test(`recall quality: ${scenario.name}`, async () => { + const db = await initSmriti(":memory:"); + try { + await seedScenario(db, scenario); + + for (const probe of scenario.probes) { + const { score } = await runProbe(db, scenario, probe, { fast: true }); + + expect(score.recall, `recall for "${probe.query}" (${probe.description})`).toBe(1); + if (score.precision !== null) { + expect(score.precision, `precision for "${probe.query}" (${probe.description})`).toBe(1); + } + expect(score.substringOk, `substring check for "${probe.query}" (${probe.description})`).toBe(true); + } + } finally { + await closeDb(); + } + }); +} diff --git a/test/store.test.ts b/test/store.test.ts new file mode 100644 index 0000000..ce0e8dd --- /dev/null +++ b/test/store.test.ts @@ -0,0 +1,33 @@ +/** + * test/store.test.ts + * + * Verifies closeDb()/closeQmdStore() dispose the QMD store's LlamaCpp/Llama + * backend, not just the SQLite handle — otherwise a fresh Store created on + * the next initSmriti() call (e.g. the daemon's per-flush open) can overlap + * with the still-loaded previous instance and leak memory between flushes. + */ +import { test, expect } from "bun:test"; +import { initSmriti, closeDb } from "../src/db"; +import { getQmdStore } from "../src/store"; + +test("closeDb disposes the QMD store's LLM backend, not just the SQLite handle", async () => { + await initSmriti(":memory:"); + const store = getQmdStore(); + + let disposeCalls = 0; + if (store.internal.llm) { + (store.internal.llm as any).dispose = async () => { disposeCalls++; }; + } + + await closeDb(); + + expect(disposeCalls).toBe(1); + expect(() => getQmdStore()).toThrow(); +}); + +test("closeDb is safe to call again after initSmriti() re-opens", async () => { + await initSmriti(":memory:"); + await closeDb(); + await initSmriti(":memory:"); + await closeDb(); // should not throw +}); diff --git a/test/team-segmented.test.ts b/test/team-segmented.test.ts index cbc9727..cc1aeb1 100644 --- a/test/team-segmented.test.ts +++ b/test/team-segmented.test.ts @@ -17,12 +17,12 @@ import type { KnowledgeUnit } from "../src/team/types"; let db: Database; -beforeAll(() => { - db = initSmriti(":memory:"); +beforeAll(async () => { + db = await initSmriti(":memory:"); }); -afterAll(() => { - closeDb(); +afterAll(async () => { + await closeDb(); }); // ============================================================================= diff --git a/test/team.test.ts b/test/team.test.ts index daa2c36..e1e82eb 100644 --- a/test/team.test.ts +++ b/test/team.test.ts @@ -2,43 +2,85 @@ * test/team.test.ts - Tests for team sharing pipeline utilities */ -import { test, expect } from "bun:test"; +import { test, expect, beforeAll, afterAll } from "bun:test"; import { isValidCategory } from "../src/categorize/schema"; import { parseFrontmatter } from "../src/team/sync"; +import { mergeCategories, readConfig, writeConfig, exportCustomCategories } from "../src/team/config"; import { initSmriti, closeDb } from "../src/db"; import type { Database } from "bun:sqlite"; +import { mkdirSync, rmSync } from "fs"; +import { join } from "path"; +import { tmpdir } from "os"; // ============================================================================= // Setup // ============================================================================= -const db: Database = initSmriti(":memory:"); +let db: Database; +let tmpDir: string; + +beforeAll(async () => { + db = await initSmriti(":memory:"); + tmpDir = join(tmpdir(), `smriti-test-${Date.now()}`); + mkdirSync(tmpDir, { recursive: true }); +}); + +afterAll(async () => { + await closeDb(); + try { rmSync(tmpDir, { recursive: true }); } catch {} +}); // ============================================================================= -// Tag Parsing Tests +// Tag Parsing Tests — #1 // ============================================================================= -test("parseFrontmatter extracts tags array", () => { +test("parseFrontmatter parses tags array into string[]", () => { const input = `--- tags: ["project", "project/dependency", "decision/tooling"] --- Body content here`; const parsed = parseFrontmatter(input); - expect(parsed.meta.tags).toBe(`["project", "project/dependency", "decision/tooling"]`); + expect(Array.isArray(parsed.meta.tags)).toBe(true); + expect(parsed.meta.tags).toEqual(["project", "project/dependency", "decision/tooling"]); expect(parsed.body).toContain("Body content here"); }); -test("parseFrontmatter extracts multiple fields", () => { +test("parseFrontmatter parses tags array with scalar category", () => { + const input = `--- +category: project +tags: ["project", "project/dependency"] +--- +Body`; + + const parsed = parseFrontmatter(input); + expect(parsed.meta.category).toBe("project"); + expect(Array.isArray(parsed.meta.tags)).toBe(true); + expect(parsed.meta.tags).toEqual(["project", "project/dependency"]); +}); + +test("parseFrontmatter keeps scalar values as strings", () => { const input = `--- category: project -tags: ["a", "b"] +author: alice --- Body`; const parsed = parseFrontmatter(input); + expect(typeof parsed.meta.category).toBe("string"); expect(parsed.meta.category).toBe("project"); - expect(parsed.meta.tags).toBe(`["a", "b"]`); + expect(parsed.meta.author).toBe("alice"); +}); + +test("parseFrontmatter handles empty tags array", () => { + const input = `--- +tags: [] +--- +Body`; + + const parsed = parseFrontmatter(input); + expect(Array.isArray(parsed.meta.tags)).toBe(true); + expect((parsed.meta.tags as string[]).length).toBe(0); }); test("parseFrontmatter handles content without frontmatter", () => { @@ -52,7 +94,7 @@ test("parseFrontmatter handles content without frontmatter", () => { // Backward Compatibility Tests // ============================================================================= -test("parseFrontmatter returns single category field", () => { +test("parseFrontmatter returns single category field (no tags array)", () => { const input = `--- category: project --- @@ -60,6 +102,7 @@ Some body`; const parsed = parseFrontmatter(input); expect(parsed.meta.category).toBe("project"); + expect(parsed.meta.tags).toBeUndefined(); }); test("parseFrontmatter extracts pipeline field for segmented docs", () => { @@ -112,3 +155,95 @@ author: testuser expect(parsed.body).toContain("# Session Title"); expect(parsed.body).toContain("**user**: Hello world"); }); + +// ============================================================================= +// mergeCategories Tests — #2 +// ============================================================================= + +test("mergeCategories adds new custom categories to DB", () => { + const n = mergeCategories(db, [ + { id: "client", name: "Client-side" }, + { id: "client/web-ui", name: "Web UI", parent: "client" }, + ]); + expect(n).toBe(2); + expect(isValidCategory(db, "client")).toBe(true); + expect(isValidCategory(db, "client/web-ui")).toBe(true); +}); + +test("mergeCategories is idempotent — second call returns 0", () => { + mergeCategories(db, [{ id: "infra", name: "Infrastructure" }]); + const n = mergeCategories(db, [{ id: "infra", name: "Infrastructure" }]); + expect(n).toBe(0); +}); + +test("mergeCategories skips builtin categories", () => { + const n = mergeCategories(db, [{ id: "bug/fix", name: "Bug Fix" }]); + expect(n).toBe(0); +}); + +test("mergeCategories handles parent-before-child ordering regardless of input order", () => { + // child listed before parent in input — should still work + const n = mergeCategories(db, [ + { id: "ops/incident", name: "Incident", parent: "ops" }, + { id: "ops", name: "Operations" }, + ]); + // Both should be created; parent first despite input order + expect(isValidCategory(db, "ops")).toBe(true); + expect(isValidCategory(db, "ops/incident")).toBe(true); + expect(n).toBe(2); +}); + +test("mergeCategories skips child whose parent doesn't exist and isn't in the batch", () => { + const n = mergeCategories(db, [ + { id: "orphan/child", name: "Orphan Child", parent: "nonexistent-parent" }, + ]); + expect(n).toBe(0); + expect(isValidCategory(db, "orphan/child")).toBe(false); +}); + +// ============================================================================= +// Config Read/Write Tests — #2 +// ============================================================================= + +test("writeConfig + readConfig roundtrip", async () => { + const config = { + version: 2, + categories: [{ id: "mycat", name: "My Category" }], + allowedCategories: ["*"] as string[], + autoSync: false, + }; + await writeConfig(tmpDir, config); + const back = readConfig(tmpDir); + expect(back.version).toBe(2); + expect(back.categories).toHaveLength(1); + expect(back.categories![0]!.id).toBe("mycat"); +}); + +test("readConfig returns default when file missing", () => { + const config = readConfig(join(tmpDir, "nonexistent")); + expect(config.version).toBe(1); + expect(config.categories).toBeUndefined(); +}); + +// ============================================================================= +// exportCustomCategories Tests — #2 +// ============================================================================= + +test("exportCustomCategories returns only non-builtin categories", () => { + // client, client/web-ui, infra, ops, ops/incident were added in earlier tests + const custom = exportCustomCategories(db); + const ids = custom.map(c => c.id); + // Should include our custom ones + expect(ids).toContain("client"); + expect(ids).toContain("ops/incident"); + // Should NOT include builtins + expect(ids).not.toContain("bug/fix"); + expect(ids).not.toContain("code"); +}); + +test("exportCustomCategories includes parent field when set", () => { + const custom = exportCustomCategories(db); + const webUi = custom.find(c => c.id === "client/web-ui"); + expect(webUi).toBeDefined(); + expect(webUi!.parent).toBe("client"); +});