From b18b4ba33a4b4ee06a6e390d97254443a568feff Mon Sep 17 00:00:00 2001 From: alex anikin <60673011+anikinsasha@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:35:46 -0700 Subject: [PATCH] fix(memory): make the Directory Inventory Status column enforceable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fresh guided install warns 14 times on every SessionEnd, forever: the inventory marks 20 rows `active` while DeployCore scaffolds 6, and nothing in the release creates the other 14. Six of them are touched by no shipped code at all, so no amount of use can clear them. The Status column is the right axis and it is already there — it is just not enforced. Only the literal `active` produces drift, so every other value grants silence, and a value the row-wide regex cannot match (`dormant archive`) drops the row out of the parse entirely: absent from the missing check AND absent from the unknown-subsystem check, which turns into a spurious `unknown_on_disk` warning on any install that has the directory. - Cells are read positionally so a multi-word status is captured verbatim. - Recognised statuses are `active`, `on-demand`, `reserved`, `pending`; anything else is reported as unclassified, never exempted. - `pending` is new: absence keeps warning while a shipped reader still has no empty state. WISDOM/ and RELATIONSHIP/ are pending for that reason. - Governance rule: no row may be moved into silence while a shipped reader still consumes it, unless the same change gives that reader an empty state. - Rows re-derived against the shipped tree; writer cells that named components with no implementation in the payload corrected or dropped. - The `reserved` definition no longer cites release tooling that is not in the public payload, and moves out of the Class block: it is a Status. - The remedy text no longer proposes reclassifying the row to silence it. Fresh guided install: 14 permanent warnings -> 2, with no directory scaffolded to fake the checker quiet. An upgraded install that already has both trees on disk sees none. DeployCore is deliberately unchanged. 23 tests, bun test, green. --- .../DOCUMENTATION/Memory/MemorySystem.md | 62 +++-- .../hooks/handlers/MemoryDirIntegrity.test.ts | 219 ++++++++++++++++++ .../hooks/handlers/MemoryDirIntegrity.ts | 202 ++++++++++------ 3 files changed, 391 insertions(+), 92 deletions(-) create mode 100644 LifeOS/install/hooks/handlers/MemoryDirIntegrity.test.ts diff --git a/LifeOS/install/LIFEOS/DOCUMENTATION/Memory/MemorySystem.md b/LifeOS/install/LIFEOS/DOCUMENTATION/Memory/MemorySystem.md index af3fb29fff..f7341b8425 100755 --- a/LifeOS/install/LIFEOS/DOCUMENTATION/Memory/MemorySystem.md +++ b/LifeOS/install/LIFEOS/DOCUMENTATION/Memory/MemorySystem.md @@ -267,46 +267,56 @@ Verdict: fresh-with-misses ## Directory Inventory (authoritative) -This is the canonical list of every directory under `~/.claude/LIFEOS/MEMORY/`. The `MemoryDirIntegrity.ts` drift handler (called by `DocIntegrity.hook.ts` on SessionEnd) parses this table and warns whenever the on-disk tree contains a directory not listed here, or this table lists a directory that no longer exists. Add new memory subsystems by adding a row to this table FIRST, then creating the directory. +This is the canonical list of every directory under `~/.claude/LIFEOS/MEMORY/`. The `MemoryDirIntegrity.ts` drift handler (called by `DocIntegrity.hook.ts` on SessionEnd) parses this table and warns whenever the on-disk tree contains a directory not listed here, or this table lists a directory that is supposed to be there and is not. **Status decides whether absence is drift, and only a recognised Status value grants silence** — see § Status below. Add new memory subsystems by adding a row to this table FIRST, then creating the directory. | Directory | Class | Status | Purpose | Primary writers | |-----------|-------|--------|---------|-----------------| | `KNOWLEDGE/` | core | active | Curated knowledge archive (People / Companies / Ideas / Research) | Algorithm LEARN, KnowledgeHarvester, manual `/knowledge add` | | `WORK/` | core | active | Per-session work directories with ISA.md as source of truth | Algorithm execution, ISASync, SessionCleanup | | `LEARNING/` | core | active | Categorized learnings (SYSTEM/ALGORITHM/FAILURES/SYNTHESIS/REFLECTIONS/SIGNALS) | SatisfactionCapture, WorkCompletionLearning, SessionHarvester, FailureCapture, LearningPatternSynthesis | -| `WISDOM/` | core | active | Level-3 compounding wisdom — FRAMES/, PRINCIPLES/, META/ | Algorithm LEARN, WisdomFrameUpdater, WisdomCrossFrameSynthesizer | -| `RESEARCH/` | core | active | Agent research outputs and OSINT dossiers | Agent task completions, OSINT workflows | -| `SECURITY/` | core | active | Security audit events (blocks, confirmations, alerts) | SecurityPipeline.hook.ts | +| `WISDOM/` | core | pending | Level-3 compounding wisdom — FRAMES/, PRINCIPLES/, META/. Read at every SessionStart by `loadWisdomFrames()`; the reader has no empty state, so this row stays loud until it gets one | Algorithm LEARN, WisdomFrameUpdater, WisdomCrossFrameSynthesizer | +| `RESEARCH/` | core | on-demand | Agent research outputs and dossiers — created on first agent write | Agent task completions | +| `SECURITY/` | core | on-demand | Security audit events — created on the first StopFailure | `EventLogger.hook.ts` (StopFailure → `SECURITY/YYYY/MM/stop-failures-*.jsonl`; self-creates) | | `STATE/` | core | active | Ephemeral runtime state (algorithms, sessions, kitty, tab-titles, events.jsonl) | Many hooks; see STATE/ section | -| `OBSERVABILITY/` | core | active | Structured event/metric JSONL feeds for the Observability pipeline (NOT auto-rotated today; rotation queued with the sensor-loop iteration) | EventLogger (tool-activity/tool-failures/config-changes), TeammateIdle, observability-transport, ComputeGap, CostTracker, syslog (Pulse), HomeSensorDetector, Speedtest | -| `VOICE/` | core | active | Voice notification audit log (ElevenLabs events) | Pulse pulse.ts voice handler | -| `RELATIONSHIP/` | core | dormant archive | Daily principal↔DA interaction notes, opinions, reflections. Last write 2026-07-08; all three writers retired in the 2026-07-11 hooks-BPE pass (`RelationshipMemory.hook.ts`, `RelationshipReflect.ts`, `OpinionTracker.ts` are gone from disk). Relationship signal is carried by the autonomic memory reviewer now; the tree is kept for recall, not appended to. | (none — read-only archive) | -| `VERIFICATION/` | core | active | Cross-vendor audit findings (Forge audit mode) | CrossVendorAudit | -| `TEAMS/` | core | active | Team configuration and membership snapshots | TeammateIdle, manual writes | -| `SKILLS/` | core | active | Skill-execution telemetry log | ShadowRelease (test-shadow-release), skill instrumentation | -| `SYSTEMUPDATES/` | core | active | Architecture change history | Manual via CreateUpdate.ts | -| `PLANS/` | core | active | Implementation plan documents (multi-session) | Manual + agent writes | -| `REFERENCE/` | core | active | Reference materials and specs preserved for recall | Manual writes | +| `OBSERVABILITY/` | core | active | Structured event/metric JSONL feeds for the Observability pipeline (NOT auto-rotated today; rotation queued with the sensor-loop iteration) | EventLogger (tool-activity/tool-failures/config-changes), ComputeGap, CostTracker, syslog (Pulse) | +| `VOICE/` | core | on-demand | Voice notification audit log (ElevenLabs events) — created on the first voice notification, and only where Pulse runs | Pulse pulse.ts voice handler | +| `RELATIONSHIP/` | core | pending | Daily principal↔DA interaction notes. No shipped component writes it — the writers named in earlier revisions of this table ship in no release — but `loadRelationshipContext()` still reads it at every SessionStart, so the row stays loud until that reader is retired or given an empty state | (none — no shipped writer) | +| `VERIFICATION/` | core | on-demand | Cross-vendor audit findings (Forge audit mode) — created on the first audit | CrossVendorAudit | +| `TEAMS/` | core | reserved | Team configuration and membership snapshots. Nothing in the release writes this tree and nothing reads it; the row is kept for taxonomy stability | (none — no shipped writer) | +| `SKILLS/` | core | active | Skill-execution telemetry log | skill instrumentation | +| `SYSTEMUPDATES/` | core | on-demand | Architecture change history — created on the first entry | `CreateUpdate.ts`, `ActivityParser.ts --generate` (both self-create) | +| `PLANS/` | core | on-demand | Implementation plan documents (multi-session) — created on first write | Manual + agent writes | +| `REFERENCE/` | core | on-demand | Reference materials and specs preserved for recall — created on first write | Manual writes | | `BOOKMARKS/` | core | on-demand | External bookmark state (X/Twitter sync) — created when the skill first runs | the X bookmarks skill's PullBookmarks workflow | | `CALLS/` | core | on-demand | Outbound-call transcripts + cost ledger — created when the skill first runs | the outbound-call skill (cost ledger, transcript store) | | `GRAPH/` | core | on-demand | Memory graph (graph.json + PATTERNS.md) — created when first built | `LIFEOS/TOOLS/MemoryGraph.ts` | | `INBOX/` | core | on-demand | Email-triage sweeps state — created when the skill first runs | the email-triage skill | -| `DATA/` | core | active | Generic structured data dumps from skills | Various skills (e.g. a crime-stats skill) | -| `SCRATCHPAD/` | core | active | Ad-hoc scratch artifacts (queries, drafts, experiments) | Ad-hoc | -| `PROJECT/` | core | active | Singular per-project notes (distinct from `LIFEOS/USER/PROJECTS/`) | Ad-hoc | -| `ARCHIVE/` | core | active | Archived legacy memory content | Manual archival | +| `DATA/` | core | on-demand | Generic structured data dumps from skills — created when a skill first writes one | Various skills | +| `SCRATCHPAD/` | core | on-demand | Ad-hoc scratch artifacts (queries, drafts, experiments) — created on first write | Ad-hoc | +| `PROJECT/` | core | on-demand | Singular per-project notes (distinct from `LIFEOS/USER/PROJECTS/`) — created on first write | Ad-hoc | +| `ARCHIVE/` | core | on-demand | Archived legacy memory content — created on first archival | Manual archival | | `AUTO/` | core | reserved | Reserved capture surface — auto-memory role retired in v7.4; stub README retained for taxonomy stability | (none active) | | `RAW/` | core | reserved | Reserved capture surface — firehose role retired in v7.0; stub README retained for taxonomy stability | (none active) | | `_/` | skill-private | on-demand | Per-skill private state (e.g. sensor data, device inventory, assessment artifacts) — created when the owning skill first runs; internal schema owned by that skill | the owning skill | | `_BROWSER_STATE/` | skill-private | reserved | Legacy browser-skill profile/cookie scratch — that skill retired 2026-07-04; dir retained for taxonomy stability | (none active) | -| `PULSE_DATA/` | core | active | Pulse v2 Data Plane materialized JSON (e.g. goals.json + .meta.json) | Pulse adapters via RebuildAll | +| `PULSE_DATA/` | core | on-demand | Pulse v2 Data Plane materialized JSON (e.g. goals.json + .meta.json) — created when Pulse first rebuilds, and only where Pulse runs | Pulse adapters via RebuildAll | **Class definitions:** - **core** — top-level LifeOS subsystem; written by core hooks/pipelines; documented in this file. - **skill-private** — `_X`-prefixed directory owned by an individual skill named `_X`. Content schema is the skill's responsibility, not the core memory system. The drift hook recognizes any `_`-prefixed dir by this convention — they are NOT enumerated by name here (naming private skills in a shipping doc would leak them); full documentation lives in the owning skill's SKILL.md. -- **reserved** — directory exists in the taxonomy and ships with public releases (via `ShadowRelease.ts` FLAT_README_ROOTS) but is not currently written by any core component. Either stays reserved or gets removed in a future migration. -**Adding a memory subsystem:** Add a row above, create the directory with a one-page README, and (if it has structured frequency) add a writer reference to the Hook Integration table below. The drift hook will accept the new directory on next Stop. +**Status:** how the directory comes to exist, and therefore whether its absence is drift. The drift hook enforces this column and nothing else. + +- **active** — guided setup creates it (`DeployCore.ts` `MEMORY_SUBDIRS`). Absent after setup is real drift → **warns**. +- **on-demand** — created on first use, by a shipped writer that self-creates it or by a hand or agent write. Absence is the normal state of a fresh install → silent. +- **reserved** — nothing in the release creates it and nothing reads it; the row is kept so the taxonomy stays stable and the directory is recognised if an older install has it. Silent, and the Purpose cell must say why it is reserved. +- **pending** — a shipped reader still consumes the directory and has no empty state yet. Absence keeps **warning**: the warning is the debt, and it is cleared by the change that gives that reader an empty state, not before. + +Anything else — a misspelling, a value with a space in it, a row added before this vocabulary existed — is neither silent nor enforced, so the hook reports the row as unclassified rather than exempting it. Silence is only ever granted by an explicit recognised value. + +**Governance — reclassification.** A row may NOT be moved to a Status whose absence is silent (`on-demand` and `reserved` both are) while a shipped component still reads the directory, unless the same change ships that reader a visible empty state. Reclassifying is a way to silence the checker, and for a directory with a live reader the warning is the only signal a fresh install ever gets that the channel has no data. `WISDOM/` and `RELATIONSHIP/` are `pending` for exactly this reason: both are read at SessionStart by `LoadContext.hook.ts`, and neither reader renders anything when its tree is empty. The rule is about readers, not writers — a directory nobody reads produces nothing anyone is waiting for. + +**Adding a memory subsystem:** Add a row above with the Status that is true of it, create the directory with a one-page README, and (if it has structured frequency) add a writer reference to the Hook Integration table below. The drift hook will accept the new directory on the next SessionEnd. --- @@ -676,8 +686,9 @@ The `MemoryDirIntegrity.ts` handler (run from `DocIntegrity.hook.ts` on SessionE 1. Lists every directory under `~/.claude/LIFEOS/MEMORY/` (one level deep, excluding `.git`, `.DS_Store`, etc.) 2. Parses the Directory Inventory table in this file 3. Reports any directory on disk not in the table (**unknown subsystem**) -4. Reports any directory in the table not on disk (**missing subsystem** — only flagged for `active` rows; `reserved` rows are allowed to be empty or absent) -5. Logs to stderr with `[MemoryDirIntegrity]` tag and emits a `doc.integrity.memory_dir` event to `STATE/events.jsonl` +4. Reports any directory in the table not on disk (**missing subsystem** — flagged for `active` and `pending` rows; `on-demand` and `reserved` rows are allowed to be absent) +5. Reports any row whose Status is not one of the four recognised values (**unclassified row**) — an unrecognised value is not an exemption +6. Logs to stderr with `[MemoryDirIntegrity]` tag and emits a `doc.integrity.memory_dir` event to `STATE/events.jsonl` If the handler reports drift, the fix is to either (a) add the new subsystem to the inventory table above, or (b) remove the stray directory from disk. Drift is a soft warning; the hook never blocks. @@ -827,6 +838,13 @@ bun run ~/.claude/hooks/handlers/MemoryDirIntegrity.ts ## Migration History +**Directory Inventory — Status made enforceable** +- Every row re-derived against the shipped tree: the six directories guided setup scaffolds are `active`, everything created on first use is `on-demand`, everything nothing writes or reads is `reserved` with a stated reason, and the two rows with a live reader and no empty state are `pending`. A fresh guided install goes from 14 permanent drift warnings to 2, without scaffolding a directory to fake the checker quiet. +- Added the `pending` Status and the **Governance — reclassification** rule: a row may not be moved into silence while a shipped reader still consumes its directory, unless the same change gives that reader an empty state. +- The Status vocabulary is now defined in this file, and the checker rejects values outside it instead of exempting them. Previously any value other than `active` granted silence, so a misspelling and a deliberate exemption were indistinguishable — and a value containing a space (`dormant archive`) dropped the row out of the parse entirely, which also removed the directory from the unknown-subsystem check and produced a spurious warning on every install that had the tree on disk. +- The `reserved` definition is restated without reference to release tooling that is not in the public payload, and moved out of the Class definitions block (`reserved` is a Status value; it never appears in the Class column). +- Writer cells corrected to name components that ship: `SECURITY/` is written by `EventLogger.hook.ts`, not the `SecurityPipeline.hook.ts` the row named; `SYSTEMUPDATES/` by `CreateUpdate.ts` and `ActivityParser.ts --generate`. Names with no implementation in the payload are removed from `OBSERVABILITY/`, `TEAMS/`, `SKILLS/` and `RELATIONSHIP/`. + **2026-05-01:** Memory System v8.1 — Full Subsystem Inventory + Drift Detection - Documented all live core subsystems that had grown organically since v7.x: OBSERVABILITY/, VOICE/, RELATIONSHIP/, WISDOM/, TEAMS/, VERIFICATION/, SKILLS/, PLANS/, REFERENCE/, BOOKMARKS/, DATA/, SCRATCHPAD/, PROJECT/, ARCHIVE/. - Added the **Directory Inventory** authoritative table (used by drift hook) — every MEMORY/ subdirectory is listed with class, status, purpose, and writers. diff --git a/LifeOS/install/hooks/handlers/MemoryDirIntegrity.test.ts b/LifeOS/install/hooks/handlers/MemoryDirIntegrity.test.ts new file mode 100644 index 0000000000..fe083b987c --- /dev/null +++ b/LifeOS/install/hooks/handlers/MemoryDirIntegrity.test.ts @@ -0,0 +1,219 @@ +/** + * MemoryDirIntegrity.test.ts — inventory parse contract and drift scoping. + * + * Run: bun test LifeOS/install/hooks/handlers/MemoryDirIntegrity.test.ts + */ + +import { describe, expect, test } from 'bun:test'; +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { parseInventoryTable, computeDrift } from './MemoryDirIntegrity'; + +const SHIPPED_DOC = join(import.meta.dir, '../../LIFEOS/DOCUMENTATION/Memory/MemorySystem.md'); + +/** DeployCore.ts MEMORY_SUBDIRS — the directories guided setup creates. */ +const DEPLOY_CORE_DIRS = ['WORK', 'KNOWLEDGE', 'LEARNING', 'STATE', 'OBSERVABILITY', 'SKILLS']; + +function table(rows: string[]): string { + return [ + '## Directory Inventory (authoritative)', + '', + '| Directory | Class | Status | Purpose | Primary writers |', + '|-----------|-------|--------|---------|-----------------|', + ...rows, + '', + '## Next Section', + ].join('\n'); +} + +describe('parseInventoryTable', () => { + test('reads name, class and status', () => { + const rows = parseInventoryTable(table(['| `WORK/` | core | active | Purpose | Writer |'])); + expect(rows).toEqual([{ name: 'WORK', klass: 'core', status: 'active' }]); + }); + + test('a status with a space in it is read verbatim, not dropped', () => { + // The regression this whole parse rewrite exists for. A row-wide pattern + // anchoring the status as one word does not match this line at all, so the + // row silently leaves the inventory — and a row that is not in the + // inventory is enforced in neither direction. + const rows = parseInventoryTable(table(['| `RELATIONSHIP/` | core | dormant archive | P | W |']))!; + expect(rows.map((r) => r.name)).toEqual(['RELATIONSHIP']); + expect(rows[0].status).toBe('dormant archive'); + }); + + test('a placeholder row documents a convention and is not a directory', () => { + const rows = parseInventoryTable(table([ + '| `_/` | skill-private | on-demand | Per-skill private state | the owning skill |', + '| `WORK/` | core | active | P | W |', + ]))!; + expect(rows.map((r) => r.name)).toEqual(['WORK']); + }); + + test('returns null when the section heading is absent', () => { + expect(parseInventoryTable('# Some other doc\n\nNo inventory here.')).toBeNull(); + }); + + test('ignores the separator row and non-row lines', () => { + const rows = parseInventoryTable(table(['| `A/` | core | active | P | W |', 'prose line', '']))!; + expect(rows.length).toBe(1); + }); + + test('stops at the next section so other tables in the file are not read', () => { + const doc = table(['| `A/` | core | active | P | W |']) + + '\n| `NOT_A_ROW/` | core | active | different table | W |\n'; + expect(parseInventoryTable(doc)!.map((r) => r.name)).toEqual(['A']); + }); +}); + +describe('computeDrift — status scoping', () => { + const rows = (status: string, klass = 'core') => [{ name: 'X', klass, status }]; + + test('active rows warn when absent', () => { + expect(computeDrift(rows('active'), []).map((d) => d.kind)).toEqual(['missing_active']); + }); + + test('pending rows warn when absent — a live reader with no empty state stays loud', () => { + expect(computeDrift(rows('pending'), []).map((d) => d.kind)).toEqual(['missing_active']); + }); + + test('on-demand rows are silent when absent', () => { + expect(computeDrift(rows('on-demand'), [])).toEqual([]); + }); + + test('reserved rows are silent when absent', () => { + expect(computeDrift(rows('reserved'), [])).toEqual([]); + }); + + test('a present directory never warns', () => { + expect(computeDrift(rows('active'), ['X'])).toEqual([]); + }); + + test('an unrecognised status is reported, never exempted', () => { + // Today an unrecognised value is indistinguishable from a deliberate + // exemption, because enforcement is `status === "active"` and everything + // else falls through to silence. A typo is not a decision. + const drift = computeDrift(rows('activee'), []); + expect(drift.map((d) => d.kind)).toEqual(['unrecognised_status']); + expect(drift[0].detail).toContain('"activee"'); + }); + + test('an unrecognised status is reported even when the directory exists', () => { + // The row's absence policy is undefined either way, and the fresh install + // that has not created the directory yet is the one that needs telling. + expect(computeDrift(rows('dormant archive'), ['X']).map((d) => d.kind)).toEqual(['unrecognised_status']); + }); + + test('one row yields at most one finding', () => { + expect(computeDrift(rows('dormant archive'), []).length).toBe(1); + }); + + test('a dir on disk with no row is still reported', () => { + const drift = computeDrift(rows('active'), ['X', 'MYSTERY']); + expect(drift.map((d) => d.kind)).toEqual(['unknown_on_disk']); + expect(drift[0].detail).toContain('MEMORY/MYSTERY/'); + }); + + test('`_`-prefixed dirs on disk are recognised by convention, not by row', () => { + expect(computeDrift(rows('active'), ['X', '_NETWORK'])).toEqual([]); + }); + + test('a row whose status the parse could not classify keeps its directory recognised', () => { + // The second half of the dropped-row failure: the old parse removed the row + // from the inventory entirely, so an install that HAS the directory started + // reporting it as an unknown subsystem. Reading the row keeps that quiet + // while the status itself is what gets reported. + const drift = computeDrift(rows('dormant archive'), ['X']); + expect(drift.some((d) => d.kind === 'unknown_on_disk')).toBe(false); + }); + + test('the missing remedy does not propose flipping the row to silence it', () => { + const detail = computeDrift(rows('active'), [])[0].detail; + expect(detail).not.toContain("change the row's status to reserved"); + expect(detail).toContain('must not be reclassified to silence it'); + }); +}); + +describe('the shipped Directory Inventory', () => { + const doc = readFileSync(SHIPPED_DOC, 'utf-8'); + const inventory = parseInventoryTable(doc)!; + + test('parses', () => { + expect(inventory).not.toBeNull(); + expect(inventory.length).toBeGreaterThan(20); + }); + + test('every row carries a recognised status', () => { + const drift = computeDrift(inventory, []).filter((d) => d.kind === 'unrecognised_status'); + expect(drift.map((d) => d.detail)).toEqual([]); + }); + + test('exactly the six DeployCore directories are `active`', () => { + const active = inventory.filter((r) => r.status === 'active').map((r) => r.name).sort(); + expect(active).toEqual([...DEPLOY_CORE_DIRS].sort()); + }); + + /** The table's own pending set. Read, never hard-coded: a `pending` row is a + * debt cleared by the release that gives its reader an empty state, so the + * set shrinks by design. Pinning the names here would make doing that + * correctly break this suite, and omitting it cost nothing — exactly the + * wrong way round. */ + const pending = inventory.filter((r) => r.status === 'pending').map((r) => r.name).sort(); + + test('the pending set is the one this table documents as pending', () => { + // Keyed on the invariant that survives a flip: the table and the governance + // paragraph that explains it say the same thing. A rebaser who flips a + // Status cell without updating the paragraph gets told here; one who + // updates both stays green. + const governance = doc.match(/\*\*Governance — reclassification\.\*\*[\s\S]*?\n\n/)![0]; + const named = [...governance.matchAll(/`(\w+)\/`/g)].map((m) => m[1]).sort(); + expect(named).toEqual(pending); + }); + + test('a fresh install before setup warns about the active and pending rows only', () => { + const active = inventory.filter((r) => r.status === 'active'); + const drift = computeDrift(inventory, []); + expect(drift.every((d) => d.kind === 'missing_active')).toBe(true); + expect(drift.length).toBe(active.length + pending.length); + }); + + test('after guided setup the perpetual warnings drop to the pending rows', () => { + // The measured symptom this change exists to fix: 14 permanent warnings on + // a fresh guided install at v7.28.3. What remains is deliberate — each + // pending row is cleared by the change that gives its reader an empty state. + const drift = computeDrift(inventory, DEPLOY_CORE_DIRS); + expect(drift.length).toBe(pending.length); + expect(drift.every((d) => pending.some((n) => d.detail.includes(`MEMORY/${n}/`)))).toBe(true); + }); + + test('an upgraded install that already has the pending directories is silent', () => { + // The install that has been running since v6 has these trees on disk. It + // should see nothing at all — neither a missing-row warning nor the + // unknown-subsystem warning a dropped row produces. + expect(computeDrift(inventory, [...DEPLOY_CORE_DIRS, ...pending])).toEqual([]); + }); + + test('every reserved row states a reason in its Purpose cell', () => { + for (const row of inventory.filter((r) => r.status === 'reserved')) { + const line = doc.split('\n').find((l) => l.includes(`\`${row.name}/\``) && l.includes('| reserved |')); + expect(line).toBeDefined(); + const purpose = line!.split('|').slice(1).map((c) => c.trim())[3] ?? ''; + expect(purpose.length).toBeGreaterThan(20); + } + }); + + test('no Primary writers cell names a component the payload does not ship', () => { + // The phantom-writer class: a writer cell naming something that exists in + // no shipped file. Scoped to the writer cell rather than the whole row on + // purpose — a Purpose cell may legitimately name a retired component while + // explaining that it is retired, and that sentence is the opposite of the + // defect. Checked against the names this PR corrected, so a future edit + // that puts one back into a writer cell is caught here. + const phantoms = ['SecurityPipeline', 'TeammateIdle', 'HomeSensorDetector', 'Speedtest', 'RelationshipMemory', 'RelationshipReflect', 'OpinionTracker', 'ShadowRelease']; + for (const line of doc.split('\n')) { + if (!/^\|\s*`[^`]+`\s*\|/.test(line)) continue; + const writers = line.split('|').slice(1).map((c) => c.trim())[4] ?? ''; + for (const name of phantoms) expect(writers).not.toContain(name); + } + }); +}); diff --git a/LifeOS/install/hooks/handlers/MemoryDirIntegrity.ts b/LifeOS/install/hooks/handlers/MemoryDirIntegrity.ts index f7cf252414..871a33f750 100755 --- a/LifeOS/install/hooks/handlers/MemoryDirIntegrity.ts +++ b/LifeOS/install/hooks/handlers/MemoryDirIntegrity.ts @@ -5,11 +5,17 @@ * PURPOSE: * Keeps the canonical "Directory Inventory" table in MemorySystem.md honest * by diffing it against the actual directory tree under LIFEOS/MEMORY/. Surfaces - * drift in two directions: + * drift in three directions: * - on-disk dir not listed in inventory (unknown subsystem) - * - inventory row marked "active" with no on-disk dir (missing subsystem) + * - inventory row whose directory must already exist, with no on-disk dir + * - inventory row the table never classified (unrecognised status) * - * "reserved"-status rows are allowed to be empty or absent. + * The Status column decides whether absence is drift, and only a recognised + * value grants silence. `active` and `pending` rows must exist; `on-demand` and + * `reserved` rows may be absent; anything else — a typo, a value with a space + * in it, a row added before this vocabulary existed — is reported rather than + * exempted. Silence is a decision someone has to make on the record, because a + * checker that quietly forgets a row cannot be distinguished from a clean tree. * * TRIGGER: SessionEnd hook (called from DocIntegrity.hook.ts) * @@ -53,12 +59,29 @@ const IGNORED_FILES = new Set(['README.md', '.DS_Store']); interface InventoryRow { name: string; // e.g., "KNOWLEDGE" or "LEARNING" - klass: string; // "core" | "skill-private" | "reserved" - status: string; // "active" | "reserved" + klass: string; // "core" | "skill-private" + status: string; // as written in the table, verbatim — never normalised } +/** + * Statuses whose absence is normal, and which therefore grant silence: + * on-demand — a shipped writer creates it on first use + * reserved — nothing creates it and nothing reads it (reason required) + * + * Statuses whose absence is reported: + * active — guided setup creates it, so absence afterwards is real drift + * pending — a shipped reader still consumes it and has no empty state yet; + * the warning is the debt, and it stands until the release that + * gives that reader an empty state reclassifies the row + * + * Any other value is neither: see UNRECOGNISED handling in computeDrift. + */ +const SILENT_WHEN_ABSENT = new Set(['on-demand', 'reserved']); +const WARN_WHEN_ABSENT = new Set(['active', 'pending']); +const KNOWN_STATUS = new Set([...SILENT_WHEN_ABSENT, ...WARN_WHEN_ABSENT]); + interface DriftItem { - kind: 'unknown_on_disk' | 'missing_active' | 'inventory_unparseable'; + kind: 'unknown_on_disk' | 'missing_active' | 'unrecognised_status' | 'inventory_unparseable'; detail: string; } @@ -71,50 +94,59 @@ interface DriftItem { * |-----------|-------|--------|---------|-----------------| * | `KNOWLEDGE/` | core | active | ... | ... | * - * Each row's first column is a backtick-wrapped directory name with a - * trailing slash. Class column is core/skill-private/reserved. Status is - * active/reserved. We only care about the directory name, class, and status - * for the drift check. + * Cells are read positionally by splitting on `|` rather than matched with one + * pattern over the whole row. The difference is not cosmetic: a single pattern + * anchoring the status cell as one word silently DROPS any row whose status + * contains a space, and a dropped row is invisible in both directions — its + * absence stops being checked, and its directory on disk starts reporting as an + * unknown subsystem. Reading the cell verbatim and judging it afterwards means + * an unclassifiable row is reported as itself. + * + * Exported for tests; the parse contract is the thing most likely to break + * silently. */ -function parseInventory(): InventoryRow[] | null { - if (!existsSync(INVENTORY_DOC)) { - console.error(`${TAG} Inventory doc not found: ${INVENTORY_DOC}`); - return null; - } - - const content = readFileSync(INVENTORY_DOC, 'utf-8'); - - // Find the inventory section. We anchor on the section heading so we don't - // accidentally pick up the auto-memory-coexistence table further down. +export function parseInventoryTable(content: string): InventoryRow[] | null { + // Anchor on the section heading so we don't pick up other tables in the file. const sectionMarker = '## Directory Inventory'; const sectionStart = content.indexOf(sectionMarker); - if (sectionStart < 0) { - console.error(`${TAG} Could not find "${sectionMarker}" in inventory doc`); - return null; - } + if (sectionStart < 0) return null; const nextSection = content.indexOf('\n## ', sectionStart + sectionMarker.length); const section = nextSection > 0 ? content.slice(sectionStart, nextSection) : content.slice(sectionStart); - // Match rows: `| `NAME/` | class | status | ... | ... |` - // Tolerate variations in whitespace and the trailing slash being optional. - const rowRegex = /^\|\s*`([\w_]+)\/?`\s*\|\s*([\w-]+)\s*\|\s*([\w-]+)\s*\|/gm; - const rows: InventoryRow[] = []; - let match: RegExpExecArray | null; - while ((match = rowRegex.exec(section)) !== null) { - rows.push({ - name: match[1], - klass: match[2].trim(), - status: match[3].trim(), - }); + for (const line of section.split('\n')) { + const named = line.match(/^\|\s*`([^`]+)`\s*\|/); + if (!named) continue; + + const name = named[1].replace(/\/$/, ''); + // `_/` and friends are convention rows: they document a naming rule + // rather than a directory. The drift check recognises `_`-prefixed dirs by + // that convention (see below), so a placeholder row has nothing to check. + if (name.includes('<')) continue; + + const cells = line.split('|').slice(1).map((c) => c.trim()); + rows.push({ name, klass: cells[1] ?? '', status: cells[2] ?? '' }); } return rows; } +function parseInventory(): InventoryRow[] | null { + if (!existsSync(INVENTORY_DOC)) { + console.error(`${TAG} Inventory doc not found: ${INVENTORY_DOC}`); + return null; + } + + const rows = parseInventoryTable(readFileSync(INVENTORY_DOC, 'utf-8')); + if (rows === null) { + console.error(`${TAG} Could not find "## Directory Inventory" in inventory doc`); + } + return rows; +} + function listMemoryDirsOnDisk(): string[] { if (!existsSync(MEMORY_DIR)) { console.error(`${TAG} MEMORY dir does not exist: ${MEMORY_DIR}`); @@ -136,6 +168,70 @@ function listMemoryDirsOnDisk(): string[] { return dirs.sort(); } +/** + * Diff the inventory against the on-disk tree. Pure — the whole drift policy, + * independent of the filesystem, and the unit under test. + * + * One row yields at most one finding: a row nobody classified is reported as + * unclassified rather than also as missing, because the remedy is the same + * sentence either way and two lines for one row trains people to skim. + */ +export function computeDrift(inventory: InventoryRow[], onDisk: string[]): DriftItem[] { + const inventoryByName = new Map(); + for (const row of inventory) inventoryByName.set(row.name, row); + const onDiskSet = new Set(onDisk); + + const drift: DriftItem[] = []; + + // Direction 1: dirs on disk not in inventory. + for (const dir of onDisk) { + // Skill-private dirs are recognized by CONVENTION, not by enumeration: any + // `_`-prefixed dir is owned by the skill named `_` (see the `_X` + // convention in MemorySystem.md). They are deliberately NOT listed by name + // in the shipping inventory — naming each private skill in a public doc is + // a leak, and enumerating them here just duplicates the convention. Their + // internal schema is the owning skill's responsibility, not core memory's. + if (dir.startsWith('_')) continue; + if (!inventoryByName.has(dir)) { + drift.push({ + kind: 'unknown_on_disk', + detail: `MEMORY/${dir}/ exists but is not listed in MemorySystem.md Directory Inventory. Either add a row or remove the directory.`, + }); + } + } + + // Direction 2: rows the table never classified. Reported whether or not the + // directory exists, because the row's absence policy is undefined until + // someone writes a recognised value — and an unrecognised value that grants + // silence is indistinguishable from a deliberate exemption. + for (const row of inventory) { + if (KNOWN_STATUS.has(row.status)) continue; + drift.push({ + kind: 'unrecognised_status', + detail: `Inventory row MEMORY/${row.name}/ has status "${row.status}", which is not one of ` + + `${[...KNOWN_STATUS].map((s) => `\`${s}\``).join(', ')}. ` + + `The row is not being enforced in either direction until it carries a recognised status. ` + + `See MemorySystem.md § Directory Inventory § Status.`, + }); + } + + // Direction 3: rows whose directory must already exist but does not. + for (const row of inventory) { + if (!WARN_WHEN_ABSENT.has(row.status)) continue; + if (onDiskSet.has(row.name)) continue; + drift.push({ + kind: 'missing_active', + detail: `MEMORY/${row.name}/ does not exist on disk and the row is \`${row.status}\`. ` + + `If setup should have created it, create it or re-run the Setup scaffold step. ` + + `If nothing in this install writes it, give the row the status that is true — ` + + `but a row whose directory still has a shipped reader must not be reclassified to silence it. ` + + `See MemorySystem.md § Directory Inventory.`, + }); + } + + return drift; +} + export async function handleMemoryDirIntegrity(): Promise { const startTime = Date.now(); console.error(`${TAG} === Starting memory inventory drift check ===`); @@ -167,42 +263,8 @@ export async function handleMemoryDirIntegrity(): Promise { return; } - const inventoryByName = new Map(); - for (const row of inventory) inventoryByName.set(row.name, row); - const onDisk = listMemoryDirsOnDisk(); - const onDiskSet = new Set(onDisk); - - const drift: DriftItem[] = []; - - // Direction 1: dirs on disk not in inventory. - for (const dir of onDisk) { - // Skill-private dirs are recognized by CONVENTION, not by enumeration: any - // `_`-prefixed dir is owned by the skill named `_` (see the `_X` - // convention in MemorySystem.md). They are deliberately NOT listed by name - // in the shipping inventory — naming each private skill in a public doc is - // a leak, and enumerating them here just duplicates the convention. Their - // internal schema is the owning skill's responsibility, not core memory's. - if (dir.startsWith('_')) continue; - if (!inventoryByName.has(dir)) { - drift.push({ - kind: 'unknown_on_disk', - detail: `MEMORY/${dir}/ exists but is not listed in MemorySystem.md Directory Inventory. Either add a row or remove the directory.`, - }); - } - } - - // Direction 2: active inventory rows missing on disk. Non-active rows - // (reserved = not-yet-built; on-demand = created when the owning skill/tool - // first runs) are allowed to be absent — a fresh install has run nothing yet. - for (const row of inventory) { - if (row.status === 'active' && !onDiskSet.has(row.name)) { - drift.push({ - kind: 'missing_active', - detail: `Inventory lists MEMORY/${row.name}/ as ${row.status} but directory does not exist on disk. Either create it or change the row's status to reserved/on-demand.`, - }); - } - } + const drift = computeDrift(inventory, onDisk); // Report. if (drift.length === 0) {