From fd0733ae056a60b0babc56c3e5ed835c264fce67 Mon Sep 17 00:00:00 2001 From: Chanu Ollala Date: Wed, 1 Jul 2026 13:03:28 -0700 Subject: [PATCH] fix: removed slop --- .gitignore | 221 ---------- CLAUDE.md | 45 -- CODEX.md | 58 --- LICENSE | 21 - README.md | 232 ---------- app/__init__.py | 0 app/dashboard.py | 908 --------------------------------------- app/garbage_collector.py | 161 ------- app/graph_memory.py | 63 --- app/main.py | 250 ----------- app/mcp_server.py | 218 ---------- app/memory_manager.py | 414 ------------------ app/models.py | 173 -------- app/retrieval.py | 102 ----- app/storage.py | 248 ----------- app/token_budget.py | 63 --- docs/ARCHITECTURE.md | 182 -------- docs/ROADMAP.md | 41 -- requirements.txt | 10 - scripts/init_memory.py | 42 -- scripts/memory_hook.py | 143 ------ scripts/save_hook.py | 181 -------- scripts/setup.sh | 184 -------- tests/__init__.py | 0 tests/conftest.py | 4 - tests/test_memory.py | 590 ------------------------- 26 files changed, 4554 deletions(-) delete mode 100644 .gitignore delete mode 100644 CLAUDE.md delete mode 100644 CODEX.md delete mode 100644 LICENSE delete mode 100644 README.md delete mode 100644 app/__init__.py delete mode 100644 app/dashboard.py delete mode 100644 app/garbage_collector.py delete mode 100644 app/graph_memory.py delete mode 100644 app/main.py delete mode 100644 app/mcp_server.py delete mode 100644 app/memory_manager.py delete mode 100644 app/models.py delete mode 100644 app/retrieval.py delete mode 100644 app/storage.py delete mode 100644 app/token_budget.py delete mode 100644 docs/ARCHITECTURE.md delete mode 100644 docs/ROADMAP.md delete mode 100644 requirements.txt delete mode 100644 scripts/init_memory.py delete mode 100644 scripts/memory_hook.py delete mode 100644 scripts/save_hook.py delete mode 100755 scripts/setup.sh delete mode 100644 tests/__init__.py delete mode 100644 tests/conftest.py delete mode 100644 tests/test_memory.py diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 54fe9d1..0000000 --- a/.gitignore +++ /dev/null @@ -1,221 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[codz] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py.cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -# Pipfile.lock - -# UV -# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# uv.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -# poetry.lock -# poetry.toml - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. -# https://pdm-project.org/en/latest/usage/project/#working-with-version-control -# pdm.lock -# pdm.toml -.pdm-python -.pdm-build/ - -# pixi -# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. -# pixi.lock -# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one -# in the .venv directory. It is recommended not to include this directory in version control. -.pixi - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# Redis -*.rdb -*.aof -*.pid - -# RabbitMQ -mnesia/ -rabbitmq/ -rabbitmq-data/ - -# ActiveMQ -activemq-data/ - -# SageMath parsed files -*.sage.py - -# Environments -.env -.envrc -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -# .idea/ - -# Abstra -# Abstra is an AI-powered process automation framework. -# Ignore directories containing user credentials, local state, and settings. -# Learn more at https://abstra.io/docs -.abstra/ - -# Visual Studio Code -# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore -# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore -# and can be added to the global gitignore or merged into this file. However, if you prefer, -# you could uncomment the following to ignore the entire vscode folder -# .vscode/ -# Temporary file for partial code execution -tempCodeRunnerFile.py - -# Machine-specific Claude Code settings (generated by scripts/setup.sh) -.claude/settings.json - -# Ruff stuff: -.ruff_cache/ - -# PyPI configuration file -.pypirc - -# Marimo -marimo/_static/ -marimo/_lsp/ -__marimo__/ - -# Streamlit -.streamlit/secrets.toml diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 464fd52..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,45 +0,0 @@ -# CLAUDE.md — Working in this repository - -This file tells Claude how to contribute to PersistentMemoryforAgents. - -## Core principles - -- **Readable over clever.** Prefer clear variable names and short functions over terse one-liners. -- **Modular by design.** Each file in `app/` has a single responsibility. Do not let concerns bleed across modules. -- **No paid APIs.** Do not introduce OpenAI, Anthropic, Cohere, or any other cloud AI API. All ML must run locally or be approximated with standard libraries. -- **FastAPI + stdlib first.** Prefer FastAPI and Python standard libraries. If adding a new package, justify it and add it to `requirements.txt`. -- **No overengineering.** Do not add abstract base classes, factory patterns, or plugin systems unless the complexity clearly demands them. Three readable functions beat a premature abstraction. - -## Making changes - -1. Find the relevant module in `app/`. Each file owns one concern (see `docs/ARCHITECTURE.md` for the component map). -2. Add or update a test in `tests/test_memory.py` for every new behavior. -3. If you change a public API endpoint, a model field, or the scoring formula, update `docs/ARCHITECTURE.md`. -4. Write short, factual commit messages ("add BM25 retrieval", not "implement sophisticated search algorithm"). - -## Running the server - -```bash -pip install -r requirements.txt -uvicorn app.main:app --reload -``` - -API docs at `http://localhost:8000/docs`. - -## Running tests - -```bash -pytest tests/ -v -``` - -## Scoring changes - -`retrieval.py:composite_score` drives every retrieval result and every GC decision. Changing its weights is a significant change. Test against the full suite and explain the tradeoff in the PR. - -## What to avoid - -- Circular imports between `app/` modules -- Business logic in `app/main.py` route functions — delegate to `MemoryManager` -- Catching broad `Exception` silently -- Leaving `# TODO` stubs without a follow-up task -- Adding `print()` debug statements to committed code diff --git a/CODEX.md b/CODEX.md deleted file mode 100644 index 506420b..0000000 --- a/CODEX.md +++ /dev/null @@ -1,58 +0,0 @@ -# CODEX.md — Coding agent contribution guide - -This document orients automated coding agents (GitHub Copilot, Claude, Cursor, etc.) contributing to PersistentMemoryforAgents. - -## Project goals - -Build a lightweight, local-first memory layer that any AI agent can use to persist, retrieve, and manage memories across long-running sessions — without cloud APIs or proprietary dependencies. - -## Architecture rules - -| Rule | Reason | -|------|--------| -| One module, one responsibility | Keeps diffs small and reviewable | -| No cross-module side effects | `storage.py` does not know about `retrieval.py` | -| Pydantic for all data boundaries | Validate at the edge, not scattered across callsites | -| Thread-safe storage | `MemoryStore` uses `threading.RLock` — do not bypass it | -| No global mutable state outside `MemoryManager` | `main.py` holds exactly one `MemoryManager` instance | - -## Module responsibilities - -| Module | Owns | Does not own | -|--------|------|--------------| -| `models.py` | Pydantic schemas | Business logic | -| `storage.py` | CRUD on the in-memory dict | Scoring, retrieval | -| `retrieval.py` | TF-IDF search + composite scoring | Storage mutations | -| `token_budget.py` | Token counting + budget allocation | Retrieval, ranking | -| `graph_memory.py` | Tag/entity graph traversal | Memory mutations | -| `garbage_collector.py` | Tier demotion, archival, deletion | Query-time retrieval | -| `memory_manager.py` | Orchestrates all components | Component internals | -| `main.py` | HTTP routing and request parsing | Business logic | - -## Testing expectations - -- Every new endpoint → at least one happy-path test and one error/404 test. -- Every scoring change → a test asserting rank order (higher importance = higher rank for equal queries). -- Every GC rule change → a test with a manufactured scenario that exercises the specific threshold. -- Run `pytest tests/ -v` before committing. All tests must pass. - -## Style guidelines - -- Max line length: 100 characters. -- Type hints on all public functions and method signatures. -- No comments explaining *what* the code does — only *why* when non-obvious (hidden invariant, workaround, subtle constraint). -- No docstrings on trivial getters and setters. -- Import order: stdlib → third-party → local, each group separated by a blank line. -- Prefer `datetime.now(timezone.utc)` over deprecated `datetime.utcnow()`. - -## Safe refactor rules - -1. Do not rename public `MemoryEntry` fields without a migration plan — clients depend on the field names. -2. Do not change scoring weights in `composite_score` without updating `docs/ARCHITECTURE.md` and adding a rank-order test. -3. Do not change GC thresholds (`PROMOTION_THRESHOLD`, etc.) without updating `docs/ARCHITECTURE.md`. -4. Do not add required fields to `AddMemoryRequest` without providing a default value. -5. Do not change `MemoryType` enum values without updating existing stored data or providing a migration. - -## Roadmap priorities - -The current focus is **v0.2 — Persistence**. When choosing between two valid approaches, prefer the one that makes swapping in a SQLite backend easiest. Storage logic is intentionally isolated in `storage.py` for this reason. See `docs/ROADMAP.md` for the full plan. diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 74516ad..0000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2026 Shreyas Kommuri - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/README.md b/README.md deleted file mode 100644 index a541158..0000000 --- a/README.md +++ /dev/null @@ -1,232 +0,0 @@ -# PersistentMemoryforAgents - -A self-hosted memory server for AI agents. Stores facts, decisions, and context across sessions in a four-tier hierarchy — working, episodic, semantic, archived — with TF-IDF retrieval, composite scoring, and automatic garbage collection. - -Works standalone via REST or as an [MCP server](#mcp--claude-code) wired directly into Claude Code. - ---- - -## How it works - -Every memory has a score computed from four factors: - -``` -score = 0.4 × semantic_similarity (TF-IDF cosine vs. query) - + 0.3 × importance (user-supplied, 0–1) - + 0.2 × recency (exp decay over age in hours) - + 0.1 × access_frequency (log-normalized hit count) -``` - -This score drives both retrieval ranking and the garbage collector's tier decisions. High-scoring memories get promoted toward `working`; low-scoring ones demote toward `archived` and eventually get deleted. - ---- - -## Quick start - -```bash -git clone https://github.com/shreyaskommuri/PersistentMemoryforAgents -cd PersistentMemoryforAgents -python3 -m venv .venv && source .venv/bin/activate -pip install -r requirements.txt -uvicorn app.main:app --reload -``` - -Open `http://localhost:8000/docs` for the interactive API. - ---- - -## MCP + Claude Code - -The MCP server lets Claude Code call `remember`, `recall`, `load_context`, `forget`, and `seed_project` natively during conversations. Memories persist to `~/.pma_store.db` across sessions. - -**Add to `.claude/settings.json`:** - -```json -{ - "mcpServers": { - "persistent-memory": { - "command": "python3", - "args": ["/absolute/path/to/PersistentMemoryforAgents/app/mcp_server.py"], - "env": { - "PMA_NAMESPACE": "my-project" - } - } - } -} -``` - -Set `PMA_NAMESPACE` per project so different workspaces don't share memories. - -**Available tools:** - -| Tool | What it does | -|------|-------------| -| `load_context` | Returns a token-budgeted context window of the most relevant memories. Call at session start. | -| `remember` | Saves a memory with importance, tags, linked entities, and memory type. | -| `recall` | TF-IDF search over stored memories. Returns scored results. | -| `forget` | Deletes a memory by ID prefix. | -| `seed_project` | Seeds memories from project docs (CLAUDE.md, README.md, ARCHITECTURE.md, ROADMAP.md). | -| `memory_stats` | Shows tier counts and token usage for the current namespace. | - ---- - -## Memory tiers - -| Tier | Analogy | Max idle age | -|------|---------|-------------| -| `working` | L1 cache | 1 hour | -| `episodic` | L2 cache | 24 hours | -| `semantic` | RAM | 7 days | -| `archived` | Disk | Indefinite | - -The garbage collector (`POST /gc`) promotes hot memories up and demotes stale ones down. Use `GET /memory/gc/preview` to see what it would do before running it. - ---- - -## API reference - -**Memories** - -| Method | Endpoint | Description | -|--------|----------|-------------| -| `POST` | `/memories` | Add a memory. `?namespace=` scopes it to a project. | -| `GET` | `/memories` | List all memories. `?namespace=` filters by project. | -| `GET` | `/memories/search` | TF-IDF search. `?q=`, `?namespace=`, `?limit=`, `?memory_type=` | -| `GET` | `/memories/context` | Token-budgeted context window for agent injection. `?q=`, `?token_budget=` | -| `GET` | `/memories/export` | Export all memories as a JSON snapshot. `?namespace=` to export one project. | -| `POST` | `/memories/import` | Import a JSON snapshot. `?skip_existing=true`, `?namespace=` to override. | -| `GET` | `/memories/{id}` | Fetch one memory (increments access count). | -| `DELETE` | `/memories/{id}` | Delete a memory. | -| `GET` | `/memories/{id}/linked` | Graph-linked memories (shared tags or entities). | - -**Graph** - -| Method | Endpoint | Description | -|--------|----------|-------------| -| `GET` | `/graph/{entity}` | Traverse the entity graph from a tag or entity name. | - -**Garbage collection** - -| Method | Endpoint | Description | -|--------|----------|-------------| -| `POST` | `/gc` | Run the garbage collector — promotes, demotes, archives, deletes. | -| `GET` | `/memory/gc/preview` | Dry-run: see every GC decision and reason without applying it. | - -**Observability** - -| Method | Endpoint | Description | -|--------|----------|-------------| -| `GET` | `/stats` | Total count, by-tier breakdown, token usage. | -| `GET` | `/memory/stats` | Detailed per-tier stats + GC pressure indicator. | -| `GET` | `/memory/inspect/{id}` | Score breakdown and GC prediction for one memory. | -| `GET` | `/memory/lineage/{id}` | Full event history: creates, accesses, promotions, demotions. | - ---- - -## Usage examples - -**Add a memory** -```bash -curl -X POST "http://localhost:8000/memories?namespace=myproject" \ - -H "Content-Type: application/json" \ - -d '{ - "content": "Use async/await for all database calls — sync calls block the event loop.", - "importance": 0.9, - "tags": ["python", "async"], - "linked_entities": ["database", "event-loop"] - }' -``` - -**Search** -```bash -curl "http://localhost:8000/memories/search?q=database+async&namespace=myproject&limit=5" -``` - -**Get a context window for agent injection** -```bash -curl "http://localhost:8000/memories/context?q=database+performance&token_budget=2048&namespace=myproject" -``` - -**Export a namespace snapshot** -```bash -curl "http://localhost:8000/memories/export?namespace=myproject" > backup.json -``` - -**Restore from snapshot** -```bash -curl -X POST "http://localhost:8000/memories/import?namespace=myproject" \ - -H "Content-Type: application/json" \ - -d @backup.json -``` - -**Preview GC decisions before running** -```bash -curl http://localhost:8000/memory/gc/preview | python3 -m json.tool -``` - ---- - -## Memory schema - -```json -{ - "id": "3f2a1b4c-...", - "content": "string", - "memory_type": "working | episodic | semantic | archived", - "importance": 0.0, - "tags": ["string"], - "linked_entities": ["string"], - "namespace": "default", - "token_count": 12, - "access_count": 3, - "created_at": "2024-01-01T00:00:00Z", - "accessed_at": "2024-01-01T01:00:00Z", - "metadata": {} -} -``` - ---- - -## Configuration - -| Variable | Default | Description | -|----------|---------|-------------| -| `PMA_STORAGE` | `sqlite` | Backend: `sqlite` for durable storage, `memory` for in-process only (tests) | -| `PMA_DB_PATH` | `~/.pma_store.db` | SQLite database file path | -| `PMA_NAMESPACE` | `default` | Namespace for MCP server — set per project in `.claude/settings.json` | - ---- - -## Running tests - -```bash -pytest tests/ -v -``` - -Tests use `PMA_STORAGE=memory` automatically (set in `tests/conftest.py`) so they never touch the real database. - ---- - -## Architecture - -See [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) for the component map and data flow. - -## Roadmap - -See [docs/ROADMAP.md](docs/ROADMAP.md). Currently at v0.2 (SQLite persistence, export/import). Next: v0.3 dense embeddings with `sentence-transformers`. - ---- - -## Tech stack - -- [FastAPI](https://fastapi.tiangolo.com/) + [Pydantic v2](https://docs.pydantic.dev/) -- [SQLAlchemy](https://www.sqlalchemy.org/) — SQLite backend -- [scikit-learn](https://scikit-learn.org/) — TF-IDF vectorization -- [MCP](https://modelcontextprotocol.io/) — Claude Code integration -- [pytest](https://pytest.org/) + [httpx](https://www.python-httpx.org/) - ---- - -## License - -MIT diff --git a/app/__init__.py b/app/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/app/dashboard.py b/app/dashboard.py deleted file mode 100644 index ba5c837..0000000 --- a/app/dashboard.py +++ /dev/null @@ -1,908 +0,0 @@ -HTML = """ - - - -PersistentMemory - - - - -
-
- PersistentMemory -
-
Memories
-
- Activity -
-
-
-
- - idle - — memories - — tokens - - -
-
- -
- - -
-
Tiers
-
-
-
-
All
-
- — -
-
-
-
-
-
Working
-
L1 · max 1 h
-
-
- — -
-
-
-
-
-
Episodic
-
L2 · max 24 h
-
-
- — -
-
-
-
-
-
Semantic
-
RAM · max 7 d
-
-
- — -
-
-
-
-
-
Archived
-
Disk · unlimited
-
-
- — -
- -
GC Pressure
-
-
— memories would change tier
-
-
-
-
-
- - -
- - -
-
- - -
-
-
-
Loading…
-
-
-
-
Loading…
-
-
- - -
-
-
- Hook injections — click a memory chip to jump to it -
- -
-
-
Loading…
-
-
- -
-
- -
- - - -""" diff --git a/app/garbage_collector.py b/app/garbage_collector.py deleted file mode 100644 index 80169ab..0000000 --- a/app/garbage_collector.py +++ /dev/null @@ -1,161 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from datetime import datetime, timezone -from typing import Optional - -from .models import GCAction, GCStats, MemoryEntry, MemoryType, ScoreBreakdown -from .retrieval import build_score_breakdown -from .storage import MemoryStore - -# Hours a memory may sit at near-zero composite score before forced demotion. -# This is a backstop for memories that score so low recency can't save them — -# not a primary GC driver. High-importance memories will score above thresholds -# and be kept regardless of age. -TIER_MAX_AGE_HOURS: dict[MemoryType, int] = { - MemoryType.working: 4, - MemoryType.episodic: 72, - MemoryType.semantic: 30 * 24, -} - -# Score thresholds driving tier changes. -# Note: max achievable GC score (no query) = 0.3 + 0.2 + 0.1 = 0.6. -PROMOTION_THRESHOLD = 0.45 -DEMOTION_THRESHOLD = 0.25 -ARCHIVE_THRESHOLD = 0.10 -DELETE_THRESHOLD = 0.05 - -_TIER_ORDER = [ - MemoryType.working, - MemoryType.episodic, - MemoryType.semantic, - MemoryType.archived, -] - - -def _promote(t: MemoryType) -> MemoryType: - return _TIER_ORDER[max(0, _TIER_ORDER.index(t) - 1)] - - -def _demote(t: MemoryType) -> MemoryType: - return _TIER_ORDER[min(len(_TIER_ORDER) - 1, _TIER_ORDER.index(t) + 1)] - - -@dataclass -class _Decision: - """Internal GC decision record. Carries everything needed for lifecycle logging.""" - memory_id: str - content_preview: str - from_tier: MemoryType - token_count: int - new_tier: Optional[MemoryType] # None means deletion - action: GCAction - reason: str - breakdown: ScoreBreakdown - - -class GarbageCollector: - def __init__(self, store: MemoryStore) -> None: - self._store = store - - def _analyze_entry(self, entry: MemoryEntry) -> _Decision: - """Pure analysis — no side effects. Returns the decision that GC would make.""" - breakdown = build_score_breakdown(entry) - score = breakdown.composite - age_hours = breakdown.age_hours - - # Age-based demotion only fires when the composite score is already low. - # This prevents age from overriding genuinely high-value memories whose - # recency component has decayed but importance/frequency keep them useful. - max_age = TIER_MAX_AGE_HOURS.get(entry.memory_type) - if max_age and age_hours > max_age and score <= DEMOTION_THRESHOLD: - new_tier = _demote(entry.memory_type) - action = GCAction.archive if new_tier == MemoryType.archived else GCAction.demote - reason = ( - f"Idle {age_hours:.1f}h exceeds {entry.memory_type} max age of {max_age}h " - f"and score {score:.3f} ≤ demotion threshold" - ) - return self._decision(entry, new_tier, action, reason, breakdown) - - if score >= PROMOTION_THRESHOLD and entry.memory_type != MemoryType.working: - new_tier = _promote(entry.memory_type) - reason = ( - f"Score {score:.3f} ≥ promotion threshold {PROMOTION_THRESHOLD} " - f"(importance={entry.importance:.2f}, recency={breakdown.recency:.2f})" - ) - return self._decision(entry, new_tier, GCAction.promote, reason, breakdown) - - if score <= DELETE_THRESHOLD and entry.memory_type == MemoryType.archived: - reason = f"Score {score:.3f} ≤ delete threshold {DELETE_THRESHOLD}, already archived" - return self._decision(entry, None, GCAction.delete, reason, breakdown) - - if score <= ARCHIVE_THRESHOLD and entry.memory_type != MemoryType.archived: - reason = f"Score {score:.3f} ≤ archive threshold {ARCHIVE_THRESHOLD}" - return self._decision( - entry, MemoryType.archived, GCAction.archive, reason, breakdown - ) - - if ( - score <= DEMOTION_THRESHOLD - and entry.memory_type not in (MemoryType.working, MemoryType.archived) - ): - new_tier = _demote(entry.memory_type) - reason = f"Score {score:.3f} ≤ demotion threshold {DEMOTION_THRESHOLD}" - return self._decision(entry, new_tier, GCAction.demote, reason, breakdown) - - reason = f"Score {score:.3f} within normal range for {entry.memory_type} tier" - return self._decision(entry, entry.memory_type, GCAction.keep, reason, breakdown) - - @staticmethod - def _decision( - entry: MemoryEntry, - new_tier: Optional[MemoryType], - action: GCAction, - reason: str, - breakdown: ScoreBreakdown, - ) -> _Decision: - return _Decision( - memory_id=entry.id, - content_preview=entry.content[:80], - from_tier=entry.memory_type, - token_count=entry.token_count, - new_tier=new_tier, - action=action, - reason=reason, - breakdown=breakdown, - ) - - def analyze(self, memory_id: str) -> Optional[_Decision]: - """Analyze a single memory with no side effects.""" - entry = self._store.get(memory_id) - return self._analyze_entry(entry) if entry else None - - def preview(self) -> list[_Decision]: - """Dry-run over all memories — no side effects.""" - return [self._analyze_entry(e) for e in self._store.all()] - - def run(self) -> tuple[GCStats, list[_Decision]]: - """Apply GC decisions and return stats + the decisions that were made.""" - stats = GCStats(promoted=0, demoted=0, archived=0, deleted=0) - decisions = [self._analyze_entry(e) for e in self._store.all()] - - for d in decisions: - if d.action == GCAction.keep: - continue - entry = self._store.get(d.memory_id) - if not entry: - continue - if d.action in (GCAction.promote, GCAction.demote, GCAction.archive): - entry.memory_type = d.new_tier - self._store.update(entry) - if d.action == GCAction.promote: - stats.promoted += 1 - elif d.action == GCAction.demote: - stats.demoted += 1 - else: - stats.archived += 1 - elif d.action == GCAction.delete: - self._store.delete(d.memory_id) - stats.deleted += 1 - - return stats, decisions diff --git a/app/graph_memory.py b/app/graph_memory.py deleted file mode 100644 index 248f4c8..0000000 --- a/app/graph_memory.py +++ /dev/null @@ -1,63 +0,0 @@ -from __future__ import annotations - -from collections import defaultdict - -from .models import GraphNeighbors, MemoryEntry -from .storage import MemoryStore - - -class GraphMemory: - """ - Implicit tag- and entity-indexed memory graph. - - Edges are derived at query time — no separate adjacency store. - Two memories are neighbors if they share at least one tag or linked entity. - """ - - def __init__(self, store: MemoryStore) -> None: - self._store = store - - def neighbors(self, entity: str) -> GraphNeighbors: - """Return all memories touching this entity/tag and their related nodes.""" - tag_index, entity_index = self._build_indexes() - key = entity.lower() - memory_ids = set(entity_index.get(key, [])) | set(tag_index.get(key, [])) - - memories: list[MemoryEntry] = [] - related: set[str] = set() - for mid in memory_ids: - entry = self._store.get(mid) - if entry: - memories.append(entry) - related.update(e.lower() for e in entry.linked_entities) - related.update(t.lower() for t in entry.tags) - related.discard(key) - - return GraphNeighbors( - entity=entity, memories=memories, related_entities=sorted(related) - ) - - def linked_memories(self, entry: MemoryEntry) -> list[MemoryEntry]: - """Return memories that share at least one tag or entity with the given entry.""" - keys = {t.lower() for t in entry.tags} | {e.lower() for e in entry.linked_entities} - if not keys: - return [] - - tag_index, entity_index = self._build_indexes() - related_ids: set[str] = set() - for k in keys: - related_ids.update(tag_index.get(k, [])) - related_ids.update(entity_index.get(k, [])) - related_ids.discard(entry.id) - - return [e for mid in related_ids if (e := self._store.get(mid)) is not None] - - def _build_indexes(self) -> tuple[dict[str, list[str]], dict[str, list[str]]]: - tag_index: dict[str, list[str]] = defaultdict(list) - entity_index: dict[str, list[str]] = defaultdict(list) - for entry in self._store.all(): - for tag in entry.tags: - tag_index[tag.lower()].append(entry.id) - for entity in entry.linked_entities: - entity_index[entity.lower()].append(entry.id) - return tag_index, entity_index diff --git a/app/main.py b/app/main.py deleted file mode 100644 index 45e5e9c..0000000 --- a/app/main.py +++ /dev/null @@ -1,250 +0,0 @@ -from __future__ import annotations - -from typing import Optional - -import json -from pathlib import Path - -from fastapi import FastAPI, HTTPException, Query -from fastapi.responses import HTMLResponse - -from .dashboard import HTML -from .memory_manager import MemoryManager -from .models import ( - AddMemoryRequest, - ContextRequest, - ContextResponse, - DetailedStats, - GCPreview, - GCStats, - GraphNeighbors, - MemoryEntry, - MemoryInspect, - MemoryLineage, - MemorySearchResult, - MemoryType, - SearchQuery, - SnapshotImportResult, -) - -app = FastAPI( - title="PersistentMemoryforAgents", - description="Adaptive memory runtime for long-running AI agents.", - version="0.1.0", -) - -_STORE_PATH = Path.home() / ".pma_store.json" -_ACTIVITY_PATH = Path.home() / ".pma_activity.json" - -manager = MemoryManager() -manager._store.load_from_file(str(_STORE_PATH)) - - -def _save() -> None: - manager._store.save_to_file(str(_STORE_PATH)) - - -@app.get("/", response_class=HTMLResponse, include_in_schema=False) -def dashboard() -> str: - return HTML - - -@app.post("/reload") -def reload_store() -> dict: - manager._store.load_from_file(str(_STORE_PATH)) - return {"loaded": manager._store.count()} - - -@app.post("/memories/recount") -def recount_tokens() -> dict: - updated = manager.recount_tokens() - _save() - return {"updated": updated} - - -@app.get("/activity") -def activity(limit: int = Query(default=20, ge=1, le=50)) -> list[dict]: - try: - log = json.loads(_ACTIVITY_PATH.read_text()) - return log[:limit] - except Exception: - return [] - - -@app.get("/health") -def health() -> dict: - return {"status": "ok"} - - -@app.get("/stats") -def stats() -> dict: - return manager.stats() - - -# ── Observability dashboard ──────────────────────────────────────────────── - - -@app.get("/memory/stats", response_model=DetailedStats) -def detailed_stats() -> DetailedStats: - return manager.detailed_stats() - - -@app.get("/memory/inspect/{memory_id}", response_model=MemoryInspect) -def inspect_memory(memory_id: str) -> MemoryInspect: - result = manager.inspect(memory_id) - if not result: - raise HTTPException(status_code=404, detail="Memory not found") - return result - - -@app.get("/memory/gc/preview", response_model=GCPreview) -def gc_preview() -> GCPreview: - return manager.gc_preview() - - -@app.get("/memory/lineage/{memory_id}", response_model=MemoryLineage) -def memory_lineage(memory_id: str) -> MemoryLineage: - result = manager.lineage(memory_id) - if not result: - raise HTTPException(status_code=404, detail="No lifecycle events found for this memory") - return result - - -# ── Memories ──────────────────────────────────────────────────────────────── # - - -@app.post("/memories", response_model=MemoryEntry, status_code=201) -def add_memory( - req: AddMemoryRequest, - namespace: str = Query(default="default", description="Memory namespace / project scope"), -) -> MemoryEntry: - req.namespace = namespace - entry = manager.add(req) - _save() - return entry - - -@app.get("/memories/search", response_model=list[MemorySearchResult]) -def search_memories( - q: str = Query(..., description="Search query text"), - memory_type: Optional[MemoryType] = Query(default=None), - tags: Optional[str] = Query(default=None, description="Comma-separated tag filter"), - limit: int = Query(default=10, ge=1, le=100), - min_score: float = Query(default=0.0, ge=0.0, le=1.0), - namespace: Optional[str] = Query(default="default", description="Memory namespace / project scope"), -) -> list[MemorySearchResult]: - tag_list = [t.strip() for t in tags.split(",")] if tags else None - query = SearchQuery( - query=q, - memory_types=[memory_type] if memory_type else None, - tags=tag_list, - limit=limit, - min_score=min_score, - namespace=namespace, - ) - return manager.search(query) - - -@app.get("/memories/context", response_model=ContextResponse) -def get_context( - q: Optional[str] = Query(default=None, description="Optional query to rank memories"), - token_budget: int = Query(default=4096, ge=1, le=32768), - memory_type: Optional[MemoryType] = Query(default=None), - namespace: Optional[str] = Query(default="default", description="Memory namespace / project scope"), -) -> ContextResponse: - req = ContextRequest( - query=q, - token_budget=token_budget, - memory_types=[memory_type] if memory_type else None, - namespace=namespace, - ) - return manager.get_context(req) - - -@app.get("/memories", response_model=list[MemoryEntry]) -def list_memories( - memory_type: Optional[MemoryType] = Query(default=None), - namespace: Optional[str] = Query(default=None, description="Filter by namespace (omit for all)"), -) -> list[MemoryEntry]: - return manager.list_all([memory_type] if memory_type else None, namespace=namespace) - - -# ── Snapshots ────────────────────────────────────────────────────────────── # -# Must be registered before /memories/{memory_id} or FastAPI matches them as IDs. - - -@app.get("/memories/export") -def export_memories( - namespace: Optional[str] = Query(default=None, description="Export one namespace (omit for all)"), -) -> list[dict]: - entries = manager.list_all(namespace=namespace) - return [e.model_dump(mode="json") for e in entries] - - -@app.post("/memories/import", response_model=SnapshotImportResult, status_code=200) -def import_memories( - snapshot: list[dict], - namespace: Optional[str] = Query(default=None, description="Override namespace for all imported entries"), - skip_existing: bool = Query(default=True, description="Skip entries whose ID already exists"), -) -> SnapshotImportResult: - imported = skipped = 0 - for raw in snapshot: - try: - entry = MemoryEntry.model_validate(raw) - except Exception: - skipped += 1 - continue - if namespace is not None: - entry.namespace = namespace - if skip_existing and manager._store.get(entry.id): - skipped += 1 - continue - manager._store.add(entry) - imported += 1 - if imported: - _save() - return SnapshotImportResult( - imported=imported, - skipped=skipped, - total_in_snapshot=len(snapshot), - ) - - -@app.get("/memories/{memory_id}", response_model=MemoryEntry) -def get_memory(memory_id: str) -> MemoryEntry: - entry = manager.get(memory_id) - if not entry: - raise HTTPException(status_code=404, detail="Memory not found") - return entry - - -@app.get("/memories/{memory_id}/linked", response_model=list[MemoryEntry]) -def linked_memories(memory_id: str) -> list[MemoryEntry]: - if not manager._store.get(memory_id): - raise HTTPException(status_code=404, detail="Memory not found") - return manager.linked_memories(memory_id) - - -@app.delete("/memories/{memory_id}", status_code=204) -def delete_memory(memory_id: str) -> None: - if not manager.delete(memory_id): - raise HTTPException(status_code=404, detail="Memory not found") - _save() - - -# ── Graph ─────────────────────────────────────────────────────────────────── # - - -@app.get("/graph/{entity}", response_model=GraphNeighbors) -def graph_neighbors(entity: str) -> GraphNeighbors: - return manager.graph_neighbors(entity) - - -# ── Garbage collection ────────────────────────────────────────────────────── # - - -@app.post("/gc", response_model=GCStats) -def run_gc() -> GCStats: - stats = manager.run_gc() - _save() - return stats diff --git a/app/mcp_server.py b/app/mcp_server.py deleted file mode 100644 index 56480bd..0000000 --- a/app/mcp_server.py +++ /dev/null @@ -1,218 +0,0 @@ -#!/usr/bin/env python3 -""" -MCP server for PersistentMemoryforAgents. - -Exposes memory tools Claude Code (and any MCP-compatible agent) can call -natively during conversations. Memories persist to ~/.pma_store.db (SQLite) -across sessions. - -Usage (Claude Code wires this up automatically via .mcp.json): - python3 app/mcp_server.py -""" -from __future__ import annotations - -import os -import sys -from pathlib import Path - -# Allow running directly from the project root. -sys.path.insert(0, str(Path(__file__).parent.parent)) - -from mcp.server.fastmcp import FastMCP - -from app.memory_manager import MemoryManager -from app.models import AddMemoryRequest, ContextRequest, MemoryType, SearchQuery - -PROJECT_ROOT = str(Path(__file__).parent.parent) - -# Resolve namespace once at startup: explicit env override → cwd → "default". -# Set PMA_NAMESPACE in the MCP server env block (per-project in .mcp.json) to -# pin a specific namespace; otherwise the working directory at launch is used so -# each project gets isolated episodic memories automatically. -def _resolve_namespace() -> str: - explicit = os.environ.get("PMA_NAMESPACE", "").strip() - if explicit: - return explicit - cwd = os.environ.get("PWD", "").strip() - return cwd if cwd else "default" - -NAMESPACE = _resolve_namespace() - -mcp = FastMCP( - "PersistentMemory", - instructions=( - "You have access to a persistent memory store. " - "Call load_context at the start of a session to recall relevant prior knowledge. " - "Call remember to save important facts, decisions, or conclusions. " - "Call recall to search for specific information. " - "Call forget to remove outdated or incorrect memories." - ), -) - -_manager = MemoryManager() - -# Auto-seed from project docs on first run in this namespace. -if len(_manager._store.all(namespace=NAMESPACE)) == 0: - _manager.seed_from_project(PROJECT_ROOT, namespace=NAMESPACE) - - -# ── Tools ────────────────────────────────────────────────────────────────── - - -@mcp.tool() -def remember( - content: str, - importance: float = 0.5, - memory_type: str = "episodic", - tags: list[str] = [], - linked_entities: list[str] = [], -) -> str: - """ - Save a memory for future retrieval. - - Args: - content: The text to remember. - importance: How important this is, 0.0–1.0. Higher = less likely to be GC'd. - memory_type: working | episodic | semantic | archived. Default episodic. - tags: Optional category labels (e.g. ["python", "bug"]). - linked_entities: Names of people, projects, or concepts this is about. - """ - try: - tier = MemoryType(memory_type) - except ValueError: - tier = MemoryType.episodic - - req = AddMemoryRequest( - content=content, - importance=importance, - memory_type=tier, - tags=tags, - linked_entities=linked_entities, - namespace=NAMESPACE, - ) - entry = _manager.add(req) - return f"Saved [{entry.id[:8]}] ({entry.memory_type}, importance={entry.importance})" - - -@mcp.tool() -def recall(query: str, limit: int = 5, memory_type: str = "") -> str: - """ - Search memories by semantic similarity to the query. - - Args: - query: What to search for. - limit: Max results to return (default 5). - memory_type: Optional filter: working | episodic | semantic | archived. - """ - types = None - if memory_type: - try: - types = [MemoryType(memory_type)] - except ValueError: - pass - - results = _manager.search(SearchQuery(query=query, limit=limit, memory_types=types, namespace=NAMESPACE)) - if not results: - return "No memories found." - - lines = [ - f"[{r.memory.memory_type} | score={r.score:.2f} | imp={r.memory.importance:.1f}] {r.memory.content}" - for r in results - ] - return "\n".join(lines) - - -@mcp.tool() -def load_context(query: str = "", token_budget: int = 2048) -> str: - """ - Retrieve a token-budgeted context window of the most relevant memories. - Call this at the start of a session or task to load relevant prior knowledge. - - Args: - query: Optional query to rank memories by relevance to current task. - token_budget: Max tokens to return (default 2048). - """ - req = ContextRequest(query=query or None, token_budget=token_budget, namespace=NAMESPACE) - resp = _manager.get_context(req) - - if not resp.memories: - return "Memory store is empty." - - lines = [ - f"[{m.memory_type} | imp={m.importance:.1f}] {m.content}" - for m in resp.memories - ] - header = ( - f"Loaded {len(resp.memories)} memories " - f"({resp.total_tokens} tokens, {resp.budget_used:.0%} of budget used):" - ) - return header + "\n" + "\n".join(lines) - - -@mcp.tool() -def forget(memory_id_prefix: str) -> str: - """ - Delete a memory by its ID (or the first few characters of it). - - Args: - memory_id_prefix: Full ID or unique prefix (e.g. "a3f1b2c4"). - """ - exact = _manager._store.get(memory_id_prefix) - if exact and exact.namespace == NAMESPACE: - _manager.delete(exact.id) - return f"Deleted [{exact.id[:8]}]: {exact.content[:60]}" - - matches = [e for e in _manager.list_all(namespace=NAMESPACE) if e.id.startswith(memory_id_prefix)] - if not matches: - return f"No memory found with ID prefix '{memory_id_prefix}'." - if len(matches) > 1: - return f"Ambiguous: {len(matches)} memories share that prefix. Use more characters." - - _manager.delete(matches[0].id) - return f"Deleted [{matches[0].id[:8]}]: {matches[0].content[:60]}" - - -@mcp.tool() -def seed_project(project_path: str = "") -> str: - """ - Seed the memory store from a project's documentation files. - Reads CLAUDE.md, CODEX.md, docs/ARCHITECTURE.md, docs/ROADMAP.md, README.md - and saves key sections as semantic memories. Safe to re-run — skips duplicates. - - Args: - project_path: Absolute path to the project root. Defaults to this project. - """ - root = project_path or PROJECT_ROOT - count = _manager.seed_from_project(root, namespace=NAMESPACE) - if count == 0: - return "No new memories added (already seeded or no docs found)." - return f"Seeded {count} memories from {root}" - - -@mcp.tool() -def memory_stats() -> str: - """Show memory system stats for the current namespace: tier counts, token usage, and GC pressure.""" - entries = _manager.list_all(namespace=NAMESPACE) - by_tier: dict[str, dict] = {} - total_tokens = 0 - for e in entries: - t = e.memory_type.value - bucket = by_tier.setdefault(t, {"count": 0, "tokens": 0}) - bucket["count"] += 1 - bucket["tokens"] += e.token_count - total_tokens += e.token_count - lines = [ - f"Namespace: {NAMESPACE}", - f"Total: {len(entries)} memories | {total_tokens} tokens", - "", - "Tier breakdown:", - ] - for tier in ("working", "episodic", "semantic", "archived"): - b = by_tier.get(tier, {"count": 0, "tokens": 0}) - if b["count"] > 0: - lines.append(f" {tier:10s} {b['count']:3d} memories {b['tokens']:5d} tokens") - return "\n".join(lines) - - -if __name__ == "__main__": - mcp.run() diff --git a/app/memory_manager.py b/app/memory_manager.py deleted file mode 100644 index 26364c7..0000000 --- a/app/memory_manager.py +++ /dev/null @@ -1,414 +0,0 @@ -from __future__ import annotations - -from datetime import datetime, timezone -from typing import Optional - -from .garbage_collector import GarbageCollector, _Decision -from .graph_memory import GraphMemory -from .models import ( - AddMemoryRequest, - ContextRequest, - ContextResponse, - DetailedStats, - GCAction, - GCPreview, - GCPreviewEntry, - GCStats, - GraphNeighbors, - LifecycleEvent, - MemoryEntry, - MemoryInspect, - MemoryLineage, - MemorySearchResult, - MemoryType, - SearchQuery, - TierStats, -) -from .retrieval import Retriever, build_score_breakdown, composite_score -from .storage import MemoryStore -from .token_budget import TokenBudgetManager, count_entry_tokens - - -class MemoryManager: - """Central orchestrator. Routes all agent requests to the appropriate subsystem.""" - - def __init__(self) -> None: - self._store = MemoryStore() - self._retriever = Retriever() - self._gc = GarbageCollector(self._store) - self._graph = GraphMemory(self._store) - self._lifecycle: dict[str, list[LifecycleEvent]] = {} - - # ------------------------------------------------------------------ # - # Lifecycle tracking (internal) # - # ------------------------------------------------------------------ # - - def _record( - self, - memory_id: str, - event_type: str, - from_tier: Optional[MemoryType] = None, - to_tier: Optional[MemoryType] = None, - trigger: str = "", - score: Optional[float] = None, - ) -> None: - event = LifecycleEvent( - timestamp=datetime.now(timezone.utc), - event_type=event_type, - from_tier=from_tier, - to_tier=to_tier, - trigger=trigger, - score=score, - ) - self._lifecycle.setdefault(memory_id, []).append(event) - - # ------------------------------------------------------------------ # - # CRUD # - # ------------------------------------------------------------------ # - - def add(self, req: AddMemoryRequest) -> MemoryEntry: - entry = MemoryEntry( - content=req.content, - memory_type=req.memory_type, - importance=req.importance, - tags=req.tags, - linked_entities=req.linked_entities, - namespace=req.namespace, - metadata=req.metadata, - ) - entry.token_count = count_entry_tokens(entry) - self._store.add(entry) - self._record(entry.id, "created", to_tier=entry.memory_type, trigger="user") - return entry - - def get(self, memory_id: str) -> Optional[MemoryEntry]: - entry = self._store.get(memory_id) - if entry: - entry.access_count += 1 - entry.accessed_at = datetime.now(timezone.utc) - self._store.update(entry) - self._record( - memory_id, "accessed", - from_tier=entry.memory_type, to_tier=entry.memory_type, - trigger="user", - ) - return entry - - def delete(self, memory_id: str) -> bool: - entry = self._store.get(memory_id) - if entry: - self._record(memory_id, "deleted", from_tier=entry.memory_type, trigger="user") - return self._store.delete(memory_id) - - def list_all( - self, - memory_types: Optional[list[MemoryType]] = None, - namespace: Optional[str] = None, - ) -> list[MemoryEntry]: - return self._store.all(memory_types, namespace=namespace) - - # ------------------------------------------------------------------ # - # Search # - # ------------------------------------------------------------------ # - - def search(self, query: SearchQuery) -> list[MemorySearchResult]: - corpus = self._store.all(namespace=query.namespace) - results = self._retriever.search( - query=query.query, - corpus=corpus, - tags=query.tags, - memory_types=query.memory_types, - limit=query.limit, - min_score=query.min_score, - ) - for result in results: - result.memory.access_count += 1 - result.memory.accessed_at = datetime.now(timezone.utc) - self._store.update(result.memory) - return results - - # ------------------------------------------------------------------ # - # Context window assembly # - # ------------------------------------------------------------------ # - - def get_context(self, req: ContextRequest) -> ContextResponse: - budget = TokenBudgetManager(total_budget=req.token_budget) - corpus = self._store.all(req.memory_types, namespace=req.namespace) - - if req.query: - results = self._retriever.search( - query=req.query, - corpus=corpus, - memory_types=req.memory_types, - limit=len(corpus) or 1, - ) - ordered = [r.memory for r in results] - else: - ordered = sorted(corpus, key=composite_score, reverse=True) - - selected = budget.allocate(ordered) - total_tokens = sum(m.token_count or count_entry_tokens(m) for m in selected) - return ContextResponse( - memories=selected, - total_tokens=total_tokens, - budget_used=budget.usage_fraction(selected), - ) - - # ------------------------------------------------------------------ # - # Graph # - # ------------------------------------------------------------------ # - - def graph_neighbors(self, entity: str) -> GraphNeighbors: - return self._graph.neighbors(entity) - - def linked_memories(self, memory_id: str) -> list[MemoryEntry]: - entry = self._store.get(memory_id) - if not entry: - return [] - return self._graph.linked_memories(entry) - - # ------------------------------------------------------------------ # - # Garbage collection # - # ------------------------------------------------------------------ # - - def run_gc(self) -> GCStats: - stats, decisions = self._gc.run() - for d in decisions: - if d.action != GCAction.keep: - self._record( - d.memory_id, - d.action.value, - from_tier=d.from_tier, - to_tier=d.new_tier, - trigger=f"gc — {d.reason}", - score=d.breakdown.composite, - ) - return stats - - # ------------------------------------------------------------------ # - # Observability # - # ------------------------------------------------------------------ # - - def inspect(self, memory_id: str) -> Optional[MemoryInspect]: - entry = self._store.get(memory_id) - if not entry: - return None - decision = self._gc.analyze(memory_id) - breakdown = build_score_breakdown(entry) - return MemoryInspect( - memory=entry, - score_breakdown=breakdown, - gc_action=decision.action, - gc_reason=decision.reason, - predicted_tier=decision.new_tier, - ) - - def gc_preview(self) -> GCPreview: - decisions = self._gc.preview() - to_promote: list[GCPreviewEntry] = [] - to_demote: list[GCPreviewEntry] = [] - to_archive: list[GCPreviewEntry] = [] - to_delete: list[GCPreviewEntry] = [] - to_keep: list[GCPreviewEntry] = [] - token_delta = 0 - - for d in decisions: - entry = GCPreviewEntry( - memory_id=d.memory_id, - content_preview=d.content_preview, - current_tier=d.from_tier, - predicted_tier=d.new_tier, - action=d.action, - reason=d.reason, - score=d.breakdown.composite, - score_breakdown=d.breakdown, - ) - if d.action == GCAction.promote: - to_promote.append(entry) - elif d.action == GCAction.demote: - to_demote.append(entry) - elif d.action == GCAction.archive: - to_archive.append(entry) - if d.from_tier != MemoryType.archived: - token_delta += d.token_count - elif d.action == GCAction.delete: - to_delete.append(entry) - token_delta += d.token_count - else: - to_keep.append(entry) - - total = len(decisions) - affected = len(to_promote) + len(to_demote) + len(to_archive) + len(to_delete) - summary = ( - f"GC would affect {affected} of {total} memories: " - f"{len(to_promote)} promoted, {len(to_demote)} demoted, " - f"{len(to_archive)} archived, {len(to_delete)} deleted. " - f"Would free {token_delta} tokens from active tiers." - ) - return GCPreview( - to_promote=to_promote, - to_demote=to_demote, - to_archive=to_archive, - to_delete=to_delete, - to_keep=to_keep, - total_affected=affected, - token_delta=token_delta, - summary=summary, - ) - - def lineage(self, memory_id: str) -> Optional[MemoryLineage]: - events = self._lifecycle.get(memory_id) - if not events: - return None - - entry = self._store.get(memory_id) - if entry: - current_tier = entry.memory_type - else: - # Memory was deleted — reconstruct tier from last event before deletion - deleted = [e for e in events if e.event_type == "deleted"] - current_tier = deleted[-1].from_tier if deleted else MemoryType.archived - - created = next((e for e in events if e.event_type == "created"), None) - if created: - ct = created.timestamp - if ct.tzinfo is None: - ct = ct.replace(tzinfo=timezone.utc) - age_hours = round((datetime.now(timezone.utc) - ct).total_seconds() / 3600.0, 2) - else: - age_hours = 0.0 - - return MemoryLineage( - memory_id=memory_id, - current_tier=current_tier, - age_hours=age_hours, - total_promotions=sum(1 for e in events if e.event_type == "promote"), - total_demotions=sum(1 for e in events if e.event_type in ("demote", "archive")), - total_accesses=sum(1 for e in events if e.event_type == "accessed"), - events=events, - ) - - def detailed_stats(self) -> DetailedStats: - all_entries = self._store.all() - empty_tier = TierStats(count=0, total_tokens=0, avg_score=0.0) - - if not all_entries: - return DetailedStats( - total=0, - total_tokens=0, - by_tier={t.value: empty_tier for t in MemoryType}, - gc_pressure=0, - avg_composite_score=0.0, - ) - - tier_buckets: dict[str, list[float]] = {t.value: [] for t in MemoryType} - tier_tokens: dict[str, int] = {t.value: 0 for t in MemoryType} - all_scores: list[float] = [] - - for entry in all_entries: - score = composite_score(entry) - tier_buckets[entry.memory_type.value].append(score) - tier_tokens[entry.memory_type.value] += entry.token_count - all_scores.append(score) - - by_tier = { - tier: TierStats( - count=len(scores), - total_tokens=tier_tokens[tier], - avg_score=round(sum(scores) / len(scores), 4) if scores else 0.0, - ) - for tier, scores in tier_buckets.items() - } - - decisions = self._gc.preview() - gc_pressure = sum(1 for d in decisions if d.action != GCAction.keep) - - return DetailedStats( - total=len(all_entries), - total_tokens=sum(e.token_count for e in all_entries), - by_tier=by_tier, - gc_pressure=gc_pressure, - avg_composite_score=round(sum(all_scores) / len(all_scores), 4), - ) - - def seed_from_project(self, project_root: str, namespace: str = "default") -> int: - """ - Read project docs and save key sections as semantic memories. - Returns the number of memories created. - Idempotent — skips sections already stored within the same namespace. - """ - from pathlib import Path - - root = Path(project_root) - docs = [ - (root / "CLAUDE.md", 0.95, ["guidelines", "meta"]), - (root / "CODEX.md", 0.90, ["guidelines", "meta"]), - (root / "docs" / "ARCHITECTURE.md", 0.95, ["architecture"]), - (root / "docs" / "ROADMAP.md", 0.85, ["roadmap"]), - (root / "README.md", 0.80, ["overview"]), - ] - - existing_prefixes = {e.content[:80] for e in self._store.all(namespace=namespace)} - count = 0 - - for doc_path, importance, tags in docs: - if not doc_path.exists(): - continue - for section in self._split_markdown(doc_path.read_text()): - if len(section) < 60: - continue - if section[:80] in existing_prefixes: - continue - self.add(AddMemoryRequest( - content=section[:600], - memory_type=MemoryType.semantic, - importance=importance, - tags=tags + [doc_path.stem.lower()], - namespace=namespace, - metadata={"source": str(doc_path), "seeded": True}, - )) - existing_prefixes.add(section[:80]) - count += 1 - - return count - - @staticmethod - def _split_markdown(text: str) -> list[str]: - """Split markdown into sections at ## headings.""" - sections: list[str] = [] - current: list[str] = [] - for line in text.splitlines(): - if line.startswith("## ") and current: - sections.append("\n".join(current).strip()) - current = [line] - else: - current.append(line) - if current: - sections.append("\n".join(current).strip()) - return sections - - def recount_tokens(self) -> int: - """Recompute token_count for every memory using the current tokenizer. - - Run this once after upgrading to tiktoken to fix counts stored under - the old word-heuristic. Returns the number of memories updated. - """ - updated = 0 - for entry in self._store.all(): - new_count = count_entry_tokens(entry) - if new_count != entry.token_count: - entry.token_count = new_count - self._store.update(entry) - updated += 1 - return updated - - def stats(self) -> dict: - all_entries = self._store.all() - by_type = {t.value: 0 for t in MemoryType} - for entry in all_entries: - by_type[entry.memory_type.value] += 1 - return { - "total": len(all_entries), - "by_type": by_type, - "total_tokens": sum(e.token_count for e in all_entries), - } diff --git a/app/models.py b/app/models.py deleted file mode 100644 index a7d090a..0000000 --- a/app/models.py +++ /dev/null @@ -1,173 +0,0 @@ -from __future__ import annotations - -import uuid -from datetime import datetime, timezone -from enum import Enum -from typing import Any, Optional - -from pydantic import BaseModel, Field - - -class MemoryType(str, Enum): - working = "working" - episodic = "episodic" - semantic = "semantic" - archived = "archived" - - -class MemoryEntry(BaseModel): - id: str = Field(default_factory=lambda: str(uuid.uuid4())) - content: str - memory_type: MemoryType = MemoryType.episodic - importance: float = Field(default=0.5, ge=0.0, le=1.0) - tags: list[str] = Field(default_factory=list) - linked_entities: list[str] = Field(default_factory=list) - created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) - accessed_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) - access_count: int = 0 - token_count: int = 0 - namespace: str = "default" - metadata: dict[str, Any] = Field(default_factory=dict) - - -class AddMemoryRequest(BaseModel): - content: str - memory_type: MemoryType = MemoryType.episodic - importance: float = Field(default=0.5, ge=0.0, le=1.0) - tags: list[str] = Field(default_factory=list) - linked_entities: list[str] = Field(default_factory=list) - namespace: str = "default" - metadata: dict[str, Any] = Field(default_factory=dict) - - -class SearchQuery(BaseModel): - query: str - memory_types: Optional[list[MemoryType]] = None - tags: Optional[list[str]] = None - limit: int = Field(default=10, ge=1, le=100) - min_score: float = Field(default=0.0, ge=0.0, le=1.0) - namespace: Optional[str] = "default" - - -class MemorySearchResult(BaseModel): - memory: MemoryEntry - score: float - - -class ContextRequest(BaseModel): - query: Optional[str] = None - token_budget: int = Field(default=4096, ge=1, le=32768) - memory_types: Optional[list[MemoryType]] = None - namespace: Optional[str] = "default" - - -class ContextResponse(BaseModel): - memories: list[MemoryEntry] - total_tokens: int - budget_used: float - - -class GCStats(BaseModel): - promoted: int - demoted: int - archived: int - deleted: int - - -class GraphNeighbors(BaseModel): - entity: str - memories: list[MemoryEntry] - related_entities: list[str] - - -# ── Observability models ─────────────────────────────────────────────────── - - -class GCAction(str, Enum): - promote = "promote" - demote = "demote" - archive = "archive" - delete = "delete" - keep = "keep" - - -class ScoreBreakdown(BaseModel): - """Component-level decomposition of composite_score for a single memory.""" - semantic_sim: float - importance: float - recency: float - access_frequency: float - composite: float - age_hours: float - - -class MemoryInspect(BaseModel): - """Full diagnostic view of a single memory: score breakdown + GC prediction.""" - memory: MemoryEntry - score_breakdown: ScoreBreakdown - gc_action: GCAction - gc_reason: str - predicted_tier: Optional[MemoryType] - - -class GCPreviewEntry(BaseModel): - memory_id: str - content_preview: str - current_tier: MemoryType - predicted_tier: Optional[MemoryType] - action: GCAction - reason: str - score: float - score_breakdown: ScoreBreakdown - - -class GCPreview(BaseModel): - """Dry-run result: what the GC would do and why, without modifying anything.""" - to_promote: list[GCPreviewEntry] - to_demote: list[GCPreviewEntry] - to_archive: list[GCPreviewEntry] - to_delete: list[GCPreviewEntry] - to_keep: list[GCPreviewEntry] - total_affected: int - token_delta: int - summary: str - - -class TierStats(BaseModel): - count: int - total_tokens: int - avg_score: float - - -class DetailedStats(BaseModel): - total: int - total_tokens: int - by_tier: dict[str, TierStats] - gc_pressure: int - avg_composite_score: float - - -class LifecycleEvent(BaseModel): - timestamp: datetime - event_type: str - from_tier: Optional[MemoryType] = None - to_tier: Optional[MemoryType] = None - trigger: str = "" - score: Optional[float] = None - - -class MemoryLineage(BaseModel): - """Full audit trail of tier migrations and access events for a memory.""" - memory_id: str - current_tier: MemoryType - age_hours: float - total_promotions: int - total_demotions: int - total_accesses: int - events: list[LifecycleEvent] - - -class SnapshotImportResult(BaseModel): - imported: int - skipped: int - total_in_snapshot: int diff --git a/app/retrieval.py b/app/retrieval.py deleted file mode 100644 index 5e0eb2d..0000000 --- a/app/retrieval.py +++ /dev/null @@ -1,102 +0,0 @@ -from __future__ import annotations - -import math -from datetime import datetime, timezone -from typing import Optional - -import numpy as np -from sklearn.feature_extraction.text import TfidfVectorizer -from sklearn.metrics.pairwise import cosine_similarity - -from .models import MemoryEntry, MemorySearchResult, MemoryType, ScoreBreakdown - -# Exponential decay rate per hour. Half-life ≈ 6.9 hours. -_DECAY_LAMBDA = 0.1 - - -def recency_score(entry: MemoryEntry) -> float: - """Exponential decay based on hours since last access.""" - now = datetime.now(timezone.utc) - accessed = entry.accessed_at - if accessed.tzinfo is None: - accessed = accessed.replace(tzinfo=timezone.utc) - age_hours = (now - accessed).total_seconds() / 3600.0 - return math.exp(-_DECAY_LAMBDA * age_hours) - - -def composite_score(entry: MemoryEntry, semantic_sim: float = 0.0) -> float: - """ - Blend four signals into a single score in [0, 1]. - Weights: semantic=0.4, importance=0.3, recency=0.2, frequency=0.1. - """ - r = recency_score(entry) - freq = min(math.log1p(entry.access_count) / 10.0, 1.0) - return 0.4 * semantic_sim + 0.3 * entry.importance + 0.2 * r + 0.1 * freq - - -def build_score_breakdown(entry: MemoryEntry, semantic_sim: float = 0.0) -> ScoreBreakdown: - """Return the full per-component score breakdown for a memory.""" - r = recency_score(entry) - freq = min(math.log1p(entry.access_count) / 10.0, 1.0) - now = datetime.now(timezone.utc) - accessed = entry.accessed_at - if accessed.tzinfo is None: - accessed = accessed.replace(tzinfo=timezone.utc) - age_hours = (now - accessed).total_seconds() / 3600.0 - total = 0.4 * semantic_sim + 0.3 * entry.importance + 0.2 * r + 0.1 * freq - return ScoreBreakdown( - semantic_sim=round(semantic_sim, 4), - importance=round(entry.importance, 4), - recency=round(r, 4), - access_frequency=round(freq, 4), - composite=round(total, 4), - age_hours=round(age_hours, 2), - ) - - -class Retriever: - def search( - self, - query: str, - corpus: list[MemoryEntry], - tags: Optional[list[str]] = None, - memory_types: Optional[list[MemoryType]] = None, - limit: int = 10, - min_score: float = 0.0, - ) -> list[MemorySearchResult]: - candidates = corpus - - if memory_types: - candidates = [e for e in candidates if e.memory_type in memory_types] - - if tags: - tag_set = {t.lower() for t in tags} - candidates = [ - e for e in candidates if tag_set.intersection(t.lower() for t in e.tags) - ] - - if not candidates: - return [] - - sims = self._tfidf_similarities(query, candidates) - - results = [] - for entry, sim in zip(candidates, sims): - score = composite_score(entry, float(sim)) - if score >= min_score: - results.append(MemorySearchResult(memory=entry, score=round(score, 4))) - - results.sort(key=lambda r: r.score, reverse=True) - return results[:limit] - - @staticmethod - def _tfidf_similarities(query: str, entries: list[MemoryEntry]) -> np.ndarray: - texts = [e.content for e in entries] - try: - vectorizer = TfidfVectorizer(stop_words="english", max_features=5000) - matrix = vectorizer.fit_transform(texts + [query]) - sims = cosine_similarity(matrix[-1], matrix[:-1]).flatten() - except ValueError: - # Corpus too small or all stop words — fall back to zeros. - sims = np.zeros(len(entries)) - return sims diff --git a/app/storage.py b/app/storage.py deleted file mode 100644 index 71d4bd8..0000000 --- a/app/storage.py +++ /dev/null @@ -1,248 +0,0 @@ -from __future__ import annotations - -import json -import os -import threading -from pathlib import Path -from typing import Optional - -from .models import MemoryEntry, MemoryType - - -class _DictStore: - """Purely in-memory store. Used when PMA_STORAGE=memory (tests, dev).""" - - def __init__(self) -> None: - self._store: dict[str, MemoryEntry] = {} - self._lock = threading.RLock() - - def add(self, entry: MemoryEntry) -> MemoryEntry: - with self._lock: - self._store[entry.id] = entry - return entry - - def get(self, memory_id: str) -> Optional[MemoryEntry]: - with self._lock: - return self._store.get(memory_id) - - def update(self, entry: MemoryEntry) -> MemoryEntry: - with self._lock: - self._store[entry.id] = entry - return entry - - def delete(self, memory_id: str) -> bool: - with self._lock: - if memory_id in self._store: - del self._store[memory_id] - return True - return False - - def all( - self, - memory_types: Optional[list[MemoryType]] = None, - namespace: Optional[str] = None, - ) -> list[MemoryEntry]: - with self._lock: - entries = list(self._store.values()) - if memory_types: - entries = [e for e in entries if e.memory_type in memory_types] - if namespace is not None: - entries = [e for e in entries if e.namespace == namespace] - return entries - - def count(self) -> int: - with self._lock: - return len(self._store) - - def clear(self) -> None: - with self._lock: - self._store.clear() - - def save_to_file(self, path: str) -> None: - pass # memory-only mode — nothing to persist - - def load_from_file(self, path: str) -> None: - pass # memory-only mode — nothing to load - - -class _SQLiteStore: - """SQLite-backed store via SQLAlchemy. Default backend.""" - - _CREATE = """ - CREATE TABLE IF NOT EXISTS memories ( - id TEXT PRIMARY KEY, - content TEXT NOT NULL, - memory_type TEXT NOT NULL, - importance REAL NOT NULL, - tags TEXT NOT NULL, - linked_entities TEXT NOT NULL, - created_at TEXT NOT NULL, - accessed_at TEXT NOT NULL, - access_count INTEGER NOT NULL DEFAULT 0, - token_count INTEGER NOT NULL DEFAULT 0, - namespace TEXT NOT NULL DEFAULT 'default', - metadata TEXT NOT NULL - ) - """ - - def __init__(self, db_url: str) -> None: - from sqlalchemy import create_engine - self._engine = create_engine(db_url, connect_args={"check_same_thread": False}) - self._lock = threading.RLock() - self._init_db() - - def _init_db(self) -> None: - from sqlalchemy import text - with self._engine.connect() as conn: - conn.execute(text(self._CREATE)) - conn.commit() - - @staticmethod - def _row_to_entry(row) -> MemoryEntry: - return MemoryEntry( - id=row[0], - content=row[1], - memory_type=row[2], - importance=row[3], - tags=json.loads(row[4]), - linked_entities=json.loads(row[5]), - created_at=row[6], - accessed_at=row[7], - access_count=row[8], - token_count=row[9], - namespace=row[10], - metadata=json.loads(row[11]), - ) - - @staticmethod - def _entry_params(entry: MemoryEntry) -> dict: - return { - "id": entry.id, - "content": entry.content, - "memory_type": entry.memory_type.value, - "importance": entry.importance, - "tags": json.dumps(entry.tags), - "linked_entities": json.dumps(entry.linked_entities), - "created_at": entry.created_at.isoformat(), - "accessed_at": entry.accessed_at.isoformat(), - "access_count": entry.access_count, - "token_count": entry.token_count, - "namespace": entry.namespace, - "metadata": json.dumps(entry.metadata), - } - - def add(self, entry: MemoryEntry) -> MemoryEntry: - from sqlalchemy import text - sql = text(""" - INSERT INTO memories - (id, content, memory_type, importance, tags, linked_entities, - created_at, accessed_at, access_count, token_count, namespace, metadata) - VALUES - (:id, :content, :memory_type, :importance, :tags, :linked_entities, - :created_at, :accessed_at, :access_count, :token_count, :namespace, :metadata) - """) - with self._lock: - with self._engine.connect() as conn: - conn.execute(sql, self._entry_params(entry)) - conn.commit() - return entry - - def get(self, memory_id: str) -> Optional[MemoryEntry]: - from sqlalchemy import text - with self._lock: - with self._engine.connect() as conn: - row = conn.execute( - text("SELECT * FROM memories WHERE id = :id"), {"id": memory_id} - ).fetchone() - return self._row_to_entry(row) if row else None - - def update(self, entry: MemoryEntry) -> MemoryEntry: - from sqlalchemy import text - sql = text(""" - UPDATE memories SET - content=:content, memory_type=:memory_type, importance=:importance, - tags=:tags, linked_entities=:linked_entities, created_at=:created_at, - accessed_at=:accessed_at, access_count=:access_count, - token_count=:token_count, namespace=:namespace, metadata=:metadata - WHERE id=:id - """) - with self._lock: - with self._engine.connect() as conn: - conn.execute(sql, self._entry_params(entry)) - conn.commit() - return entry - - def delete(self, memory_id: str) -> bool: - from sqlalchemy import text - with self._lock: - with self._engine.connect() as conn: - result = conn.execute( - text("DELETE FROM memories WHERE id = :id"), {"id": memory_id} - ) - conn.commit() - return result.rowcount > 0 - - def all( - self, - memory_types: Optional[list[MemoryType]] = None, - namespace: Optional[str] = None, - ) -> list[MemoryEntry]: - from sqlalchemy import text - conditions: list[str] = [] - params: dict = {} - if memory_types: - placeholders = ", ".join(f":t{i}" for i in range(len(memory_types))) - conditions.append(f"memory_type IN ({placeholders})") - for i, t in enumerate(memory_types): - params[f"t{i}"] = t.value - if namespace is not None: - conditions.append("namespace = :namespace") - params["namespace"] = namespace - where = f"WHERE {' AND '.join(conditions)}" if conditions else "" - with self._lock: - with self._engine.connect() as conn: - rows = conn.execute(text(f"SELECT * FROM memories {where}"), params).fetchall() - return [self._row_to_entry(r) for r in rows] - - def count(self) -> int: - from sqlalchemy import text - with self._lock: - with self._engine.connect() as conn: - return conn.execute(text("SELECT COUNT(*) FROM memories")).scalar() or 0 - - def clear(self) -> None: - from sqlalchemy import text - with self._lock: - with self._engine.connect() as conn: - conn.execute(text("DELETE FROM memories")) - conn.commit() - - def save_to_file(self, path: str) -> None: - pass # SQLite writes are immediate — nothing to flush - - def load_from_file(self, path: str) -> None: - """One-time migration from the legacy JSON store if the DB is empty.""" - if self.count() > 0: - return - try: - with open(path) as f: - data = json.load(f) - for entry_data in data.values(): - self.add(MemoryEntry.model_validate(entry_data)) - except FileNotFoundError: - pass - except Exception: - pass - - -def MemoryStore() -> "_DictStore | _SQLiteStore": - """Factory: returns the backend selected by PMA_STORAGE env var. - - PMA_STORAGE=sqlite (default) — durable SQLite file at PMA_DB_PATH - PMA_STORAGE=memory — in-process dict, no persistence (tests/dev) - """ - backend = os.environ.get("PMA_STORAGE", "sqlite") - if backend == "memory": - return _DictStore() - db_path = os.environ.get("PMA_DB_PATH", str(Path.home() / ".pma_store.db")) - return _SQLiteStore(f"sqlite:///{db_path}") diff --git a/app/token_budget.py b/app/token_budget.py deleted file mode 100644 index 6b50bcc..0000000 --- a/app/token_budget.py +++ /dev/null @@ -1,63 +0,0 @@ -from __future__ import annotations - -import math -from functools import lru_cache - -from .models import MemoryEntry, MemoryType - -# Soft token limits per tier — used by the context assembler and GC docs. -TIER_TOKEN_LIMITS: dict[MemoryType, int] = { - MemoryType.working: 2_000, - MemoryType.episodic: 8_000, - MemoryType.semantic: 32_000, - MemoryType.archived: 128_000, -} - - -@lru_cache(maxsize=1) -def _encoder(): - """Load the tiktoken cl100k_base encoder once and reuse it. - - cl100k_base is the closest public approximation to Claude's BPE tokenizer. - Falls back to a word-count heuristic if tiktoken is not installed. - """ - try: - import tiktoken - return tiktoken.get_encoding("cl100k_base") - except ImportError: - return None - - -def count_tokens(text: str) -> int: - """Count tokens in a string using tiktoken, or a word-count heuristic as fallback.""" - enc = _encoder() - if enc is not None: - return max(1, len(enc.encode(text))) - # Fallback: ~1.3 tokens per whitespace-delimited word (BPE approximation) - return max(1, math.ceil(len(text.split()) * 1.3)) - - -def count_entry_tokens(entry: MemoryEntry) -> int: - parts = [entry.content] + entry.tags + entry.linked_entities - return count_tokens(" ".join(parts)) - - -class TokenBudgetManager: - def __init__(self, total_budget: int = 4096) -> None: - self.total_budget = total_budget - - def allocate(self, memories: list[MemoryEntry]) -> list[MemoryEntry]: - """Pack memories greedily in score order until the token budget is exhausted.""" - result: list[MemoryEntry] = [] - used = 0 - for mem in memories: - tokens = mem.token_count or count_entry_tokens(mem) - if used + tokens > self.total_budget: - continue - result.append(mem) - used += tokens - return result - - def usage_fraction(self, memories: list[MemoryEntry]) -> float: - used = sum(m.token_count or count_entry_tokens(m) for m in memories) - return min(1.0, used / self.total_budget) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md deleted file mode 100644 index 40895e2..0000000 --- a/docs/ARCHITECTURE.md +++ /dev/null @@ -1,182 +0,0 @@ -# Architecture - -PersistentMemoryforAgents is an OS-inspired memory management layer for long-running AI agents. It organizes agent memory into four tiers modeled after the CPU/OS memory hierarchy, with automatic promotion, demotion, and garbage collection driven by composite importance scores. - ---- - -## Memory tiers - -| Tier | OS analogy | Approx. token limit | Max idle age | -|------|-----------|---------------------|--------------| -| `working` | L1 CPU cache | 2,000 | 4 hours | -| `episodic` | L2 CPU cache | 8,000 | 72 hours | -| `semantic` | RAM | 32,000 | 30 days | -| `archived` | Disk | Unlimited | Indefinite | - -Memories start in the tier specified at creation time and migrate automatically. A hot memory (high access frequency, high importance) climbs toward `working`; a cold one sinks toward `archived` and eventually gets deleted. - ---- - -## Composite scoring - -Every memory receives a score in **[0, 1]** computed at query time: - -``` -score = 0.4 × semantic_similarity - + 0.3 × importance - + 0.2 × recency_score - + 0.1 × access_frequency -``` - -| Component | Formula | Notes | -|-----------|---------|-------| -| `semantic_similarity` | TF-IDF cosine sim vs. query | 0.0 when no query | -| `importance` | User-supplied, in [0, 1] | Set at creation | -| `recency_score` | `exp(-0.1 × age_hours)` | Half-life ≈ 7 h | -| `access_frequency` | `min(log1p(count) / 10, 1.0)` | Log-normalized | - -Weights sum to 1.0. Changing them changes every retrieval and GC decision — see `app/retrieval.py:composite_score`. - ---- - -## Retrieval - -`Retriever` (in `app/retrieval.py`) builds a fresh TF-IDF matrix over the candidate corpus using scikit-learn's `TfidfVectorizer`, then computes cosine similarity against the query vector. Results are re-ranked with `composite_score` to balance semantic relevance with recency and importance. - -Filtering happens before vectorization: -1. Filter by `memory_type` (if specified) -2. Filter by tag intersection (if specified) -3. Compute TF-IDF over remaining corpus -4. Sort by `composite_score`, return top-k above `min_score` - ---- - -## Graph memory - -`GraphMemory` (in `app/graph_memory.py`) maintains an implicit bipartite graph: - -``` -memory ──tagged_with──> tag -memory ──links_to──> entity -``` - -Edges are derived at query time from `MemoryEntry.tags` and `MemoryEntry.linked_entities` — no separate edge store. `neighbors(entity)` returns all memories touching that tag or entity, plus the union of their other tags/entities as related nodes. This enables lateral context expansion beyond keyword matching. - ---- - -## Token budget - -`TokenBudgetManager` (in `app/token_budget.py`) enforces a configurable token ceiling when assembling a context window: - -- Token count uses `tiktoken` (`cl100k_base`) when available, falling back to `ceil(word_count × 1.3)`. -- Memories are packed greedily in descending score order until the budget is exhausted. -- The `GET /memories/context` endpoint exposes this as a one-call context-window builder for agent use. - ---- - -## Garbage collector - -`GarbageCollector` (in `app/garbage_collector.py`) runs synchronously on `POST /gc` and on every auto-save when the store exceeds 80 memories. It evaluates each memory in this order: - -1. **Age demotion** — if a memory has been idle longer than its tier's max age *and* its composite score is ≤ `DEMOTION_THRESHOLD` (0.25), move it one tier down. Score takes priority: a high-importance memory is never evicted solely because it is old. -2. **Score promotion** — if `composite_score ≥ 0.45`, promote one tier up. -3. **Score archival / deletion**: - - `score < 0.10` → move to `archived` - - `score < 0.05` AND already `archived` → delete permanently - -GC thresholds live in `app/garbage_collector.py` as module-level constants. - ---- - -## Component map - -``` -app/ -├── main.py FastAPI routes (HTTP boundary only) -├── memory_manager.py Orchestrator — the only place all components meet -├── models.py Pydantic schemas shared across all modules -├── storage.py SQLite backend (SQLAlchemy) + in-memory backend for tests -├── retrieval.py TF-IDF search + composite_score -├── graph_memory.py Tag/entity graph traversal -├── token_budget.py Token counting (tiktoken) + budget allocation -├── garbage_collector.py Tier promotion/demotion/deletion -└── mcp_server.py MCP tool server (remember/recall/forget/load_context) - -scripts/ -├── memory_hook.py UserPromptSubmit hook — injects relevant memories as context -└── save_hook.py Stop hook — auto-saves each exchange as an episodic memory -``` - -`main.py` holds a single `MemoryManager` instance. Routes parse requests, delegate to `MemoryManager`, and return responses — no business logic in routes. - ---- - -## Data flow: add → search → context - -``` -POST /memories - AddMemoryRequest → MemoryManager.add() - → count tokens (token_budget.py) - → MemoryStore.add() - ← MemoryEntry - -GET /memories/search?q=... - → MemoryManager.search() - → MemoryStore.all() (snapshot) - → Retriever.search() (TF-IDF + composite_score) - → bump access_count (MemoryStore.update) - ← list[MemorySearchResult] - -GET /memories/context?q=...&token_budget=4096 - → MemoryManager.get_context() - → Retriever.search() or score-sort - → TokenBudgetManager.allocate() - ← ContextResponse {memories, total_tokens, budget_used} -``` - ---- - -## Storage - -`MemoryStore` (in `app/storage.py`) is a factory that returns one of two backends selected by the `PMA_STORAGE` environment variable: - -- **`sqlite`** (default) — durable SQLite file at `~/.pma_store.db` via SQLAlchemy. All writes commit immediately; no explicit flush needed. -- **`memory`** — in-process `dict` protected by `threading.RLock`, used by tests and ephemeral dev runs. - -## Namespacing - -Every `MemoryEntry` carries a `namespace` field (default `"default"`). All read and write operations accept a `namespace` parameter to filter to one scope. - -In practice: -- `scripts/save_hook.py` writes episodic memories into the **cwd namespace** (the absolute path of the active project), so exchanges from different projects never mix. -- `scripts/memory_hook.py` queries the **cwd namespace** (project-specific episodic memories) and the **`"default"` namespace** (global working/semantic rules) separately, then merges the results. -- `app/mcp_server.py` resolves its namespace from `PMA_NAMESPACE` env var → `PWD` → `"default"` at startup. - -To isolate a project manually, set `PMA_NAMESPACE=/absolute/path/to/project` in the MCP server's env block in `.mcp.json`. - ---- - -## Observability layer - -Four endpoints expose the internal decision-making of the memory runtime: - -### `GET /memory/inspect/{id}` -Returns a `MemoryInspect` with: -- Full `ScoreBreakdown`: `semantic_sim`, `importance`, `recency`, `access_frequency`, `composite`, `age_hours` -- `gc_action` and `gc_reason`: what GC would do to this memory and why -- `predicted_tier`: the tier it would land in after GC - -### `GET /memory/gc/preview` -Dry-run of the garbage collector. Returns `GCPreview` with five classified lists (`to_promote`, `to_demote`, `to_archive`, `to_delete`, `to_keep`), each entry containing the score breakdown and the human-readable reason for the decision. Also returns `token_delta` (tokens freed from active tiers) and a `summary` string. **No side effects.** - -### `GET /memory/lineage/{id}` -Full audit trail for a memory: every `created`, `accessed`, `promote`, `demote`, `archive`, and `deleted` event with timestamps, tier transitions, and scores. Events are recorded by `MemoryManager` in an in-process `_lifecycle` dict. Lineage persists even after the memory is deleted. - -### `GET /memory/stats` -Richer than `GET /stats`: per-tier `TierStats` (count, total tokens, avg score), `gc_pressure` (count of memories that would change tier in the next GC run), and `avg_composite_score` across all tiers. - -### Score breakdown formula (reminder) -``` -composite = 0.4 × semantic_sim + 0.3 × importance + 0.2 × recency + 0.1 × access_frequency -``` -In GC context (no query), `semantic_sim = 0.0` → max reachable score = 0.6. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md deleted file mode 100644 index d9bc695..0000000 --- a/docs/ROADMAP.md +++ /dev/null @@ -1,41 +0,0 @@ -# Roadmap - -## v0.1 — Foundation (current) -- [x] Four-tier memory system: working / episodic / semantic / archived -- [x] TF-IDF retrieval with composite scoring (importance + recency + frequency + semantic) -- [x] Token-budget context assembly -- [x] Tag + entity graph memory with lateral traversal -- [x] Score- and age-based garbage collection with tier migration -- [x] FastAPI REST interface with OpenAPI docs -- [x] Thread-safe in-memory storage -- [x] Unit tests with FastAPI TestClient - -## v0.2 — Persistence -- [x] SQLite backend via SQLAlchemy (one-file swap for `storage.py`) -- [x] Memory snapshots: export/import to JSON — `GET /memories/export`, `POST /memories/import` -- [x] Configurable storage backend via environment variable — `PMA_STORAGE=sqlite` (default) or `memory`; DB path via `PMA_DB_PATH` - -## v0.3 — Smarter retrieval -- [ ] Local dense embeddings with `sentence-transformers` -- [ ] Hybrid BM25 + dense re-rank -- [ ] Approximate nearest-neighbor index (FAISS or `usearch`) -- [ ] Cross-session memory deduplication - -## v0.4 — Agent integration -- [x] MCP server endpoint for direct Claude Code / Claude Desktop integration -- [ ] OpenAI-compatible function-call interface (`remember`, `recall`, `forget`) -- [x] Agent session namespacing (multi-tenant memory) — `namespace` field on all memories; `PMA_NAMESPACE` env var scopes MCP server per-project -- [ ] Streaming context assembly for large budgets - -## v0.5 — Observability -- [ ] Prometheus `/metrics` endpoint -- [ ] Per-memory audit log (access history) -- [ ] GC run history and tier-migration event stream -- [ ] Dashboard (simple HTML/JS served by FastAPI) - -## v1.0 — Production -- [ ] Redis-backed store for horizontal scaling -- [ ] Multi-agent shared memory with conflict resolution -- [ ] Access control and scoped memory per agent ID -- [ ] Docker image + `docker-compose.yml` -- [ ] Full async FastAPI with async storage backend diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index c4f905a..0000000 --- a/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -fastapi>=0.111.0 -mcp>=1.0.0 -uvicorn[standard]>=0.29.0 -pydantic>=2.7.0 -scikit-learn>=1.4.0 -numpy>=1.26.0 -sqlalchemy>=2.0 -tiktoken>=0.7 -pytest>=8.2.0 -httpx>=0.27.0 diff --git a/scripts/init_memory.py b/scripts/init_memory.py deleted file mode 100644 index 2c1db6b..0000000 --- a/scripts/init_memory.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python3 -""" -Seed the memory store from project documentation. - -Run once after setup (or re-run any time to pick up doc changes). -Reads CLAUDE.md, CODEX.md, docs/ARCHITECTURE.md, docs/ROADMAP.md, README.md -and saves each section as a semantic memory in ~/.pma_store.json. - -Usage: - python3 scripts/init_memory.py - python3 scripts/init_memory.py /path/to/other/project -""" -from __future__ import annotations - -import sys -from pathlib import Path - -PROJECT_ROOT = Path(__file__).parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -STORE_PATH = Path.home() / ".pma_store.json" - - -def main() -> None: - from app.memory_manager import MemoryManager - - target = Path(sys.argv[1]) if len(sys.argv) > 1 else PROJECT_ROOT - - manager = MemoryManager() - manager._store.load_from_file(str(STORE_PATH)) - - before = manager._store.count() - count = manager.seed_from_project(str(target)) - manager._store.save_to_file(str(STORE_PATH)) - after = manager._store.count() - - print(f"Seeded {count} new memories from {target}") - print(f"Store: {before} → {after} total memories saved to {STORE_PATH}") - - -if __name__ == "__main__": - main() diff --git a/scripts/memory_hook.py b/scripts/memory_hook.py deleted file mode 100644 index c4b8b00..0000000 --- a/scripts/memory_hook.py +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env python3 -""" -UserPromptSubmit hook for Claude Code. - -On every user prompt, retrieves relevant memories from the shared store and -injects them as additionalContext so Claude sees prior knowledge automatically. - -Also auto-seeds the current project's docs (CLAUDE.md, README.md, etc.) the -first time Claude Code opens in a directory that hasn't been seen before. - -Fails silently — if the store is empty or missing, no context is injected. -""" -from __future__ import annotations - -import json -import sys -from datetime import datetime, timezone -from pathlib import Path - -PROJECT_ROOT = Path(__file__).parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -STORE_PATH = Path.home() / ".pma_store.json" -SEEN_PROJECTS_PATH = Path.home() / ".pma_seen_projects.json" -ACTIVITY_PATH = Path.home() / ".pma_activity.json" -ACTIVITY_MAX = 50 - - -def _load_seen_projects() -> set[str]: - try: - with open(SEEN_PROJECTS_PATH) as f: - return set(json.load(f)) - except Exception: - return set() - - -def _save_seen_projects(seen: set[str]) -> None: - try: - with open(SEEN_PROJECTS_PATH, "w") as f: - json.dump(sorted(seen), f) - except Exception: - pass - - -def _log_activity(prompt: str, cwd: str, memory_ids: list[str], tokens: int) -> None: - try: - try: - log = json.loads(ACTIVITY_PATH.read_text()) - except Exception: - log = [] - log.insert(0, { - "ts": datetime.now(timezone.utc).isoformat(), - "prompt": prompt[:120], - "cwd": cwd, - "count": len(memory_ids), - "tokens": tokens, - "ids": memory_ids[:20], - }) - ACTIVITY_PATH.write_text(json.dumps(log[:ACTIVITY_MAX])) - except Exception: - pass - - -def main() -> None: - try: - data = json.load(sys.stdin) - except Exception: - sys.exit(0) - - prompt = data.get("prompt", "").strip() - cwd = data.get("cwd", "").strip() - if not prompt: - sys.exit(0) - - try: - from app.memory_manager import MemoryManager - from app.models import ContextRequest - - manager = MemoryManager() - manager._store.load_from_file(str(STORE_PATH)) - - if cwd: - seen = _load_seen_projects() - if cwd not in seen: - # Seed project docs into the project-scoped namespace - count = manager.seed_from_project(cwd, namespace=cwd) - seen.add(cwd) - _save_seen_projects(seen) - - if manager._store.count() == 0: - sys.exit(0) - - # Query project-specific memories + global working/semantic memories separately, - # then merge so cross-project episodic noise doesn't bleed in. - proj_namespace = cwd if cwd else "default" - proj_resp = manager.get_context(ContextRequest( - query=prompt, token_budget=1400, namespace=proj_namespace, - )) - global_resp = manager.get_context(ContextRequest( - query=prompt, token_budget=600, namespace="default", - )) - - seen_ids: set[str] = set() - merged = [] - for m in proj_resp.memories + global_resp.memories: - if m.id not in seen_ids: - seen_ids.add(m.id) - merged.append(m) - - if not merged: - sys.exit(0) - - total_tokens = sum(m.token_count or 0 for m in merged) - - _log_activity( - prompt=prompt, - cwd=cwd, - memory_ids=[m.id for m in merged], - tokens=total_tokens, - ) - - lines = [ - f"- [{m.memory_type} | imp={m.importance:.1f}] {m.content}" - for m in merged - ] - context = ( - f"Relevant memories from prior sessions ({total_tokens} tokens):\n" - + "\n".join(lines) - ) - output = { - "hookSpecificOutput": { - "hookEventName": "UserPromptSubmit", - "additionalContext": context, - } - } - print(json.dumps(output)) - - except Exception: - pass - - -if __name__ == "__main__": - main() diff --git a/scripts/save_hook.py b/scripts/save_hook.py deleted file mode 100644 index 87fbd08..0000000 --- a/scripts/save_hook.py +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/env python3 -""" -Stop hook for Claude Code. - -Fires after every Claude response. Saves the last exchange (user prompt + -assistant response) as an episodic memory so nothing is lost to auto-compaction. - -The loop: - UserPromptSubmit hook → inject relevant memories into context - Stop hook → save this exchange back to the store - -This means compaction doesn't matter: every substantial exchange is already -in PMA before the context window fills up. -""" -from __future__ import annotations - -import hashlib -import json -import os -import sys -from pathlib import Path - -PROJECT_ROOT = Path(__file__).parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -MIN_RESPONSE_LENGTH = 250 # skip one-liners and trivial acks -MAX_PROMPT_CHARS = 200 # how much of the prompt to store -MAX_RESPONSE_CHARS = 500 # how much of the response to store -GC_THRESHOLD = 80 # run GC when store exceeds this many memories - - -def _read_last_exchange(transcript_path: str) -> tuple[str, str] | None: - """Return (user_prompt, full_assistant_response) for the most recent exchange. - - A single Claude turn produces multiple assistant entries in the transcript - (one per reasoning step / tool use). This collects all of them between - two user turns and concatenates them into one response string. - """ - try: - lines = [l.strip() for l in Path(transcript_path).read_text().splitlines() if l.strip()] - except Exception: - return None - - messages = [] - for line in lines: - try: - messages.append(json.loads(line)) - except Exception: - continue - - def extract_text(content) -> str: - if isinstance(content, str): - return content - if isinstance(content, list): - return " ".join( - block.get("text", "") - for block in content - if isinstance(block, dict) and block.get("type") == "text" - ).strip() - return "" - - # Find the last user turn that has real text - last_user_idx = None - last_user_text = "" - for i, msg in enumerate(messages): - if msg.get("type") == "user": - text = extract_text(msg.get("message", {}).get("content", "")) - if text.strip(): - last_user_idx = i - last_user_text = text - - if last_user_idx is None: - return None - - # Collect all assistant text produced after that user turn - assistant_parts = [] - for msg in messages[last_user_idx + 1:]: - if msg.get("type") == "user": - break # next user turn started — stop - if msg.get("type") == "assistant": - text = extract_text(msg.get("message", {}).get("content", "")) - if text.strip(): - assistant_parts.append(text.strip()) - - assistant_text = " ".join(assistant_parts) - return (last_user_text, assistant_text) if assistant_text else None - - -def _prompt_hash(prompt: str) -> str: - return hashlib.md5(prompt[:200].encode()).hexdigest()[:12] - - -def _extract_response_summary(response: str, max_chars: int) -> str: - """Take the first substantive paragraph from Claude's response.""" - for para in response.split("\n\n"): - para = para.strip() - if len(para) > 60: - return para[:max_chars] - return response[:max_chars] - - -def _importance(prompt: str, response: str) -> float: - combined = prompt + response - if len(combined) > 3000: - score = 0.75 - elif len(combined) > 1000: - score = 0.65 - else: - score = 0.50 - - # Boost for responses that contain real work - work_signals = ["```", "def ", "class ", "fixed", "implement", "decided", - "error", "bug", "updated", "added", "removed"] - if any(sig in combined.lower() for sig in work_signals): - score = min(0.90, score + 0.10) - - return score - - -def main() -> None: - try: - data = json.load(sys.stdin) - except Exception: - sys.exit(0) - - transcript_path = data.get("transcript_path", "") - if not transcript_path: - sys.exit(0) - - cwd = data.get("cwd", "").strip() - namespace = cwd if cwd else "default" - - exchange = _read_last_exchange(transcript_path) - if not exchange: - sys.exit(0) - - user_prompt, assistant_response = exchange - - if len(assistant_response) < MIN_RESPONSE_LENGTH: - sys.exit(0) # too short to be worth saving - - try: - from app.memory_manager import MemoryManager - from app.models import AddMemoryRequest, MemoryType - - manager = MemoryManager() - - # Dedup by prompt hash stored in metadata - ph = _prompt_hash(user_prompt) - existing_hashes = { - e.metadata.get("prompt_hash") - for e in manager._store.all(namespace=namespace) - } - if ph in existing_hashes: - sys.exit(0) - - summary = _extract_response_summary(assistant_response, MAX_RESPONSE_CHARS) - content = ( - f"User: {user_prompt[:MAX_PROMPT_CHARS].strip()}\n" - f"Claude: {summary}" - ) - - manager.add(AddMemoryRequest( - content=content, - memory_type=MemoryType.episodic, - importance=_importance(user_prompt, assistant_response), - tags=["session", "auto-saved"], - namespace=namespace, - metadata={"prompt_hash": ph, "project": cwd}, - )) - - # Periodically GC to evict low-scoring stale memories - if manager._store.count() > GC_THRESHOLD: - manager.run_gc() - - except Exception: - pass - - -if __name__ == "__main__": - main() diff --git a/scripts/setup.sh b/scripts/setup.sh deleted file mode 100755 index 4f25710..0000000 --- a/scripts/setup.sh +++ /dev/null @@ -1,184 +0,0 @@ -#!/usr/bin/env bash -# One-time setup for PersistentMemoryforAgents. -# Safe to re-run — merges into existing configs rather than overwriting. -# -# What it does: -# 1. Creates .venv and installs Python dependencies -# 2. Writes .claude/settings.json for this project (MCP server + hook) -# 3. Merges persistent-memory into ~/.mcp.json (global MCP server) -# 4. Merges the UserPromptSubmit hook into ~/.claude/settings.json (global hook) -# 5. Seeds the memory store from project docs - -set -euo pipefail - -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -VENV="$REPO_ROOT/.venv" - -# ── Colors ───────────────────────────────────────────────────────────────── -GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BOLD='\033[1m'; NC='\033[0m' -info() { echo -e " ${GREEN}✓${NC} $*"; } -pending() { echo -e " ${YELLOW}→${NC} $*"; } - -echo "" -echo -e "${BOLD}PersistentMemoryforAgents — setup${NC}" -echo " Repo: $REPO_ROOT" -echo "" - -# ── 1. Python venv ────────────────────────────────────────────────────────── -if [ -f "$VENV/bin/python3" ]; then - info "venv already exists, skipping creation" -else - pending "Creating venv..." - python3 -m venv "$VENV" - info "venv created at $VENV" -fi - -PYTHON="$VENV/bin/python3" - -pending "Installing dependencies..." -"$PYTHON" -m pip install -q -r "$REPO_ROOT/requirements.txt" -info "Dependencies installed" - -# ── 2. Project .claude/settings.json ──────────────────────────────────────── -pending "Writing .claude/settings.json..." -mkdir -p "$REPO_ROOT/.claude" -cat > "$REPO_ROOT/.claude/settings.json" << EOF -{ - "mcpServers": { - "persistent-memory": { - "type": "stdio", - "command": "$PYTHON", - "args": ["$REPO_ROOT/app/mcp_server.py"] - } - }, - "hooks": { - "UserPromptSubmit": [ - { - "matcher": "", - "hooks": [ - { - "type": "command", - "command": "$PYTHON $REPO_ROOT/scripts/memory_hook.py" - } - ] - } - ] - } -} -EOF -info "Wrote .claude/settings.json" - -# ── 3. Global ~/.mcp.json ─────────────────────────────────────────────────── -pending "Updating ~/.mcp.json..." -"$PYTHON" - "$PYTHON" "$REPO_ROOT/app/mcp_server.py" << 'PYEOF' -import json, sys, pathlib - -python_path, server_script = sys.argv[1], sys.argv[2] -mcp_path = pathlib.Path.home() / ".mcp.json" -try: - existing = json.loads(mcp_path.read_text()) if mcp_path.exists() else {} -except Exception: - existing = {} -existing.setdefault("mcpServers", {})["persistent-memory"] = { - "type": "stdio", - "command": python_path, - "args": [server_script], -} -mcp_path.write_text(json.dumps(existing, indent=2) + "\n") -PYEOF -info "Updated ~/.mcp.json" - -# ── 4. Global ~/.claude/settings.json (hook only — mcpServers not allowed) ── -pending "Updating ~/.claude/settings.json..." -"$PYTHON" - "$PYTHON" "$REPO_ROOT/scripts/memory_hook.py" << 'PYEOF' -import json, sys, pathlib - -python_path, hook_script = sys.argv[1], sys.argv[2] -settings_path = pathlib.Path.home() / ".claude" / "settings.json" -try: - existing = json.loads(settings_path.read_text()) if settings_path.exists() else {} -except Exception: - existing = {} - -hook_command = f"{python_path} {hook_script}" -new_entry = {"matcher": "", "hooks": [{"type": "command", "command": hook_command}]} - -existing.setdefault("hooks", {}).setdefault("UserPromptSubmit", []) -# Replace any prior persistent-memory hook, append the updated one. -kept = [h for h in existing["hooks"]["UserPromptSubmit"] - if "memory_hook.py" not in str(h)] -existing["hooks"]["UserPromptSubmit"] = kept + [new_entry] - -# Stop hook — save each exchange back to the store after Claude responds -save_script = sys.argv[2].replace("memory_hook.py", "save_hook.py") -save_command = f"{python_path} {save_script}" -save_entry = {"matcher": "", "hooks": [{"type": "command", "command": save_command}]} -existing.setdefault("hooks", {}).setdefault("Stop", []) -existing["hooks"]["Stop"] = [ - h for h in existing["hooks"]["Stop"] if "save_hook.py" not in str(h) -] + [save_entry] - -settings_path.write_text(json.dumps(existing, indent=2) + "\n") -PYEOF -info "Updated ~/.claude/settings.json (UserPromptSubmit + Stop hooks)" - -# ── 5. Seed memory store from project docs ────────────────────────────────── -pending "Seeding memory store from project docs..." -SEEDED=$("$PYTHON" "$REPO_ROOT/scripts/init_memory.py" 2>&1 | grep "Seeded" | head -1) -info "${SEEDED:-Memory store already seeded}" - -# ── 6. launchd service (macOS auto-start on login) ─────────────────────────── -PLIST_LABEL="com.pma.server" -PLIST_PATH="$HOME/Library/LaunchAgents/$PLIST_LABEL.plist" -UVICORN="$VENV/bin/uvicorn" -LOG_DIR="$HOME/Library/Logs/pma" - -pending "Installing launchd service..." -mkdir -p "$LOG_DIR" -cat > "$PLIST_PATH" << EOF - - - - - Label $PLIST_LABEL - ProgramArguments - - $UVICORN - app.main:app - --host 127.0.0.1 - --port 8000 - - WorkingDirectory $REPO_ROOT - RunAtLoad - KeepAlive - StandardOutPath $LOG_DIR/server.log - StandardErrorPath $LOG_DIR/server.log - EnvironmentVariables - - PATH $VENV/bin:/usr/local/bin:/usr/bin:/bin - - - -EOF - -# Load (or reload) the service -launchctl unload "$PLIST_PATH" 2>/dev/null || true -launchctl load -w "$PLIST_PATH" -info "launchd service installed — server starts automatically on login" -info "Logs: $LOG_DIR/server.log" - -# ── Done ──────────────────────────────────────────────────────────────────── -echo "" -echo -e "${BOLD}Setup complete.${NC} Restart Claude Code for changes to take effect." -echo "" -echo " Memory tools available in any project:" -echo " remember / recall / load_context / forget / memory_stats" -echo "" -echo " Context is injected automatically on every prompt." -echo " Server runs automatically on login — no manual start needed." -echo "" -echo " To check server status: curl http://localhost:8000/health" -echo " To view logs: tail -f $LOG_DIR/server.log" -echo " To stop the service: launchctl unload $PLIST_PATH" -echo "" diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 4130812..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,4 +0,0 @@ -import os - -# Force in-memory backend for all tests so they never touch ~/.pma_store.db -os.environ.setdefault("PMA_STORAGE", "memory") diff --git a/tests/test_memory.py b/tests/test_memory.py deleted file mode 100644 index f1494f2..0000000 --- a/tests/test_memory.py +++ /dev/null @@ -1,590 +0,0 @@ -import pytest -from fastapi.testclient import TestClient - -from app.main import app, manager - -client = TestClient(app) - - -@pytest.fixture(autouse=True) -def clear_store(): - manager._store.clear() - yield - manager._store.clear() - - -# ── Health / stats ───────────────────────────────────────────────────────── - - -def test_health(): - r = client.get("/health") - assert r.status_code == 200 - assert r.json()["status"] == "ok" - - -def test_stats_empty(): - r = client.get("/stats") - assert r.status_code == 200 - data = r.json() - assert data["total"] == 0 - assert "by_type" in data - - -def test_stats_after_add(): - client.post("/memories", json={"content": "hello"}) - r = client.get("/stats") - assert r.json()["total"] == 1 - - -# ── Add / get / delete ───────────────────────────────────────────────────── - - -def test_add_memory_defaults(): - r = client.post("/memories", json={"content": "The sky is blue."}) - assert r.status_code == 201 - data = r.json() - assert data["content"] == "The sky is blue." - assert data["memory_type"] == "episodic" - assert data["importance"] == 0.5 - assert data["token_count"] > 0 - - -def test_add_memory_custom_fields(): - r = client.post( - "/memories", - json={ - "content": "Paris is the capital of France.", - "memory_type": "semantic", - "importance": 0.9, - "tags": ["geography", "europe"], - "linked_entities": ["Paris", "France"], - }, - ) - assert r.status_code == 201 - data = r.json() - assert data["memory_type"] == "semantic" - assert data["importance"] == 0.9 - assert "geography" in data["tags"] - assert "France" in data["linked_entities"] - - -def test_get_memory_increments_access_count(): - mid = client.post("/memories", json={"content": "Access me."}).json()["id"] - initial = client.get(f"/memories/{mid}").json()["access_count"] - client.get(f"/memories/{mid}") - updated = client.get(f"/memories/{mid}").json()["access_count"] - assert updated > initial - - -def test_get_memory_not_found(): - r = client.get("/memories/does-not-exist") - assert r.status_code == 404 - - -def test_delete_memory(): - mid = client.post("/memories", json={"content": "Delete me."}).json()["id"] - assert client.delete(f"/memories/{mid}").status_code == 204 - assert client.get(f"/memories/{mid}").status_code == 404 - - -def test_delete_memory_not_found(): - assert client.delete("/memories/does-not-exist").status_code == 404 - - -def test_list_memories(): - client.post("/memories", json={"content": "A"}) - client.post("/memories", json={"content": "B"}) - r = client.get("/memories") - assert r.status_code == 200 - assert len(r.json()) == 2 - - -def test_list_memories_filter_by_type(): - client.post("/memories", json={"content": "Working", "memory_type": "working"}) - client.post("/memories", json={"content": "Episodic", "memory_type": "episodic"}) - r = client.get("/memories?memory_type=working") - assert r.status_code == 200 - assert all(m["memory_type"] == "working" for m in r.json()) - - -# ── Search ───────────────────────────────────────────────────────────────── - - -def test_search_returns_results(): - client.post("/memories", json={"content": "Python is a programming language."}) - client.post("/memories", json={"content": "FastAPI is a web framework for Python."}) - client.post("/memories", json={"content": "The Eiffel Tower is in Paris."}) - - r = client.get("/memories/search?q=python+programming") - assert r.status_code == 200 - results = r.json() - assert len(results) > 0 - top_contents = [res["memory"]["content"] for res in results[:2]] - assert any("Python" in c or "FastAPI" in c for c in top_contents) - - -def test_search_tag_filter(): - client.post("/memories", json={"content": "API endpoint", "tags": ["api"]}) - client.post("/memories", json={"content": "Database table", "tags": ["db"]}) - - r = client.get("/memories/search?q=endpoint&tags=api") - assert r.status_code == 200 - for res in r.json(): - assert "api" in [t.lower() for t in res["memory"]["tags"]] - - -def test_search_empty_corpus(): - r = client.get("/memories/search?q=anything") - assert r.status_code == 200 - assert r.json() == [] - - -def test_search_importance_ranking(): - client.post("/memories", json={"content": "machine learning", "importance": 0.9}) - client.post("/memories", json={"content": "machine learning", "importance": 0.1}) - - results = client.get("/memories/search?q=machine+learning").json() - assert len(results) == 2 - assert results[0]["score"] >= results[1]["score"] - - -# ── Context window ───────────────────────────────────────────────────────── - - -def test_context_respects_token_budget(): - for i in range(10): - client.post("/memories", json={"content": f"Memory number {i} with some content here."}) - - r = client.get("/memories/context?token_budget=50") - assert r.status_code == 200 - data = r.json() - assert data["total_tokens"] <= 60 # small overshoot tolerance from estimation - assert 0.0 <= data["budget_used"] <= 1.0 - - -def test_context_with_query(): - client.post("/memories", json={"content": "Python is great for data science."}) - client.post("/memories", json={"content": "I had eggs for breakfast."}) - - r = client.get("/memories/context?q=python+data") - assert r.status_code == 200 - memories = r.json()["memories"] - contents = [m["content"] for m in memories] - assert any("Python" in c for c in contents) - - -def test_context_empty_store(): - r = client.get("/memories/context") - assert r.status_code == 200 - assert r.json()["memories"] == [] - assert r.json()["total_tokens"] == 0 - - -# ── Graph memory ─────────────────────────────────────────────────────────── - - -def test_graph_neighbors_by_entity(): - client.post( - "/memories", - json={"content": "Plants photosynthesize.", "linked_entities": ["plant", "photosynthesis"]}, - ) - client.post( - "/memories", - json={"content": "Plants produce oxygen.", "linked_entities": ["plant", "oxygen"]}, - ) - - r = client.get("/graph/plant") - assert r.status_code == 200 - data = r.json() - assert data["entity"] == "plant" - assert len(data["memories"]) == 2 - assert "oxygen" in data["related_entities"] or "photosynthesis" in data["related_entities"] - - -def test_graph_neighbors_by_tag(): - client.post("/memories", json={"content": "FastAPI tutorial.", "tags": ["python", "web"]}) - client.post("/memories", json={"content": "Django tutorial.", "tags": ["python", "web"]}) - - r = client.get("/graph/python") - assert r.status_code == 200 - assert len(r.json()["memories"]) == 2 - - -def test_graph_empty_entity(): - r = client.get("/graph/nonexistent-entity") - assert r.status_code == 200 - assert r.json()["memories"] == [] - - -def test_linked_memories(): - id1 = client.post( - "/memories", - json={"content": "Alpha memory.", "tags": ["shared"]}, - ).json()["id"] - client.post("/memories", json={"content": "Beta memory.", "tags": ["shared"]}) - - r = client.get(f"/memories/{id1}/linked") - assert r.status_code == 200 - assert len(r.json()) >= 1 - - -def test_linked_memories_not_found(): - r = client.get("/memories/does-not-exist/linked") - assert r.status_code == 404 - - -# ── Garbage collector ────────────────────────────────────────────────────── - - -def test_gc_runs_and_returns_stats(): - client.post("/memories", json={"content": "Some memory.", "importance": 0.5}) - r = client.post("/gc") - assert r.status_code == 200 - data = r.json() - assert all(k in data for k in ("promoted", "demoted", "archived", "deleted")) - - -def test_gc_promotes_high_importance(): - # High importance + recently accessed → should promote from episodic toward working. - client.post( - "/memories", - json={"content": "Critical fact.", "importance": 1.0, "memory_type": "episodic"}, - ) - client.post("/gc") - r = client.get("/memories") - promoted = [m for m in r.json() if m["memory_type"] == "working"] - assert len(promoted) >= 1 - - -# ── Observability: /memory/stats ─────────────────────────────────────────── - - -def test_detailed_stats_structure(): - client.post("/memories", json={"content": "A memory.", "importance": 0.8}) - r = client.get("/memory/stats") - assert r.status_code == 200 - data = r.json() - assert "total" in data and data["total"] >= 1 - assert "total_tokens" in data - assert "by_tier" in data - assert "gc_pressure" in data - assert "avg_composite_score" in data - for tier in ("working", "episodic", "semantic", "archived"): - assert tier in data["by_tier"] - - -def test_detailed_stats_empty_store(): - r = client.get("/memory/stats") - assert r.status_code == 200 - data = r.json() - assert data["total"] == 0 - assert data["gc_pressure"] == 0 - assert data["avg_composite_score"] == 0.0 - - -def test_detailed_stats_token_count(): - client.post("/memories", json={"content": "Token counting memory.", "importance": 0.5}) - r = client.get("/memory/stats") - assert r.json()["total_tokens"] > 0 - - -# ── Observability: /memory/inspect/{id} ──────────────────────────────────── - - -def test_inspect_memory_score_breakdown(): - mid = client.post( - "/memories", - json={"content": "Important insight.", "importance": 0.9}, - ).json()["id"] - - r = client.get(f"/memory/inspect/{mid}") - assert r.status_code == 200 - data = r.json() - - # Score breakdown fields - breakdown = data["score_breakdown"] - assert "composite" in breakdown - assert "importance" in breakdown - assert "recency" in breakdown - assert "access_frequency" in breakdown - assert "age_hours" in breakdown - assert breakdown["importance"] == pytest.approx(0.9, abs=0.01) - assert 0.0 <= breakdown["composite"] <= 1.0 - - # GC decision fields - assert "gc_action" in data - assert "gc_reason" in data - assert len(data["gc_reason"]) > 0 - - -def test_inspect_memory_not_found(): - r = client.get("/memory/inspect/does-not-exist") - assert r.status_code == 404 - - -def test_inspect_memory_high_importance_predicts_promotion(): - mid = client.post( - "/memories", - json={"content": "Critical data.", "importance": 1.0, "memory_type": "episodic"}, - ).json()["id"] - r = client.get(f"/memory/inspect/{mid}") - data = r.json() - assert data["gc_action"] == "promote" - assert data["predicted_tier"] == "working" - - -def test_inspect_memory_low_importance_predicts_archive(): - mid = client.post( - "/memories", - json={"content": "Trivial note.", "importance": 0.0}, - ).json()["id"] - r = client.get(f"/memory/inspect/{mid}") - data = r.json() - assert data["gc_action"] in ("archive", "demote", "keep") - - -# ── Observability: /memory/gc/preview ───────────────────────────────────── - - -def test_gc_preview_structure(): - client.post("/memories", json={"content": "A memory.", "importance": 0.5}) - r = client.get("/memory/gc/preview") - assert r.status_code == 200 - data = r.json() - for key in ("to_promote", "to_demote", "to_archive", "to_delete", "to_keep"): - assert key in data - assert "total_affected" in data - assert "token_delta" in data - assert "summary" in data - assert len(data["summary"]) > 0 - - -def test_gc_preview_is_dry_run(): - client.post("/memories", json={"content": "High value.", "importance": 1.0, "memory_type": "episodic"}) - preview = client.get("/memory/gc/preview").json() - assert len(preview["to_promote"]) >= 1 - - # Memories should NOT have changed after the preview - memories_after = client.get("/memories").json() - assert all(m["memory_type"] == "episodic" for m in memories_after) - - -def test_gc_preview_shows_score_breakdowns(): - client.post("/memories", json={"content": "Test memory."}) - data = client.get("/memory/gc/preview").json() - all_entries = data["to_promote"] + data["to_demote"] + data["to_archive"] + data["to_keep"] - for entry in all_entries: - assert "score_breakdown" in entry - assert "reason" in entry - assert len(entry["reason"]) > 0 - - -def test_gc_preview_token_delta_nonnegative(): - client.post("/memories", json={"content": "Low value memory.", "importance": 0.0}) - data = client.get("/memory/gc/preview").json() - assert data["token_delta"] >= 0 - - -def test_gc_preview_empty_store(): - r = client.get("/memory/gc/preview") - assert r.status_code == 200 - data = r.json() - assert data["total_affected"] == 0 - assert data["token_delta"] == 0 - - -# ── Observability: /memory/lineage/{id} ──────────────────────────────────── - - -def test_lineage_records_created_event(): - mid = client.post("/memories", json={"content": "Tracked memory."}).json()["id"] - r = client.get(f"/memory/lineage/{mid}") - assert r.status_code == 200 - data = r.json() - event_types = [e["event_type"] for e in data["events"]] - assert "created" in event_types - - -def test_lineage_records_accessed_event(): - mid = client.post("/memories", json={"content": "Accessed memory."}).json()["id"] - client.get(f"/memories/{mid}") # triggers access - r = client.get(f"/memory/lineage/{mid}") - data = r.json() - event_types = [e["event_type"] for e in data["events"]] - assert "accessed" in event_types - assert data["total_accesses"] >= 1 - - -def test_lineage_records_gc_event(): - mid = client.post( - "/memories", - json={"content": "GC target.", "importance": 1.0, "memory_type": "episodic"}, - ).json()["id"] - client.post("/gc") - r = client.get(f"/memory/lineage/{mid}") - data = r.json() - event_types = [e["event_type"] for e in data["events"]] - assert "promote" in event_types - assert data["total_promotions"] >= 1 - - -def test_lineage_includes_tier_transitions(): - mid = client.post( - "/memories", - json={"content": "Promoted memory.", "importance": 1.0, "memory_type": "semantic"}, - ).json()["id"] - client.post("/gc") - r = client.get(f"/memory/lineage/{mid}") - data = r.json() - promote_events = [e for e in data["events"] if e["event_type"] == "promote"] - if promote_events: - ev = promote_events[0] - assert ev["from_tier"] is not None - assert ev["to_tier"] is not None - assert ev["score"] is not None - - -def test_lineage_not_found(): - r = client.get("/memory/lineage/does-not-exist") - assert r.status_code == 404 - - -def test_lineage_age_hours_nonnegative(): - mid = client.post("/memories", json={"content": "Age test."}).json()["id"] - data = client.get(f"/memory/lineage/{mid}").json() - assert data["age_hours"] >= 0.0 - - -# ── Namespacing ──────────────────────────────────────────────────────────── - - -def test_namespace_isolates_list(): - client.post("/memories?namespace=proj_a", json={"content": "Project A memory."}) - client.post("/memories?namespace=proj_b", json={"content": "Project B memory."}) - - a_mems = client.get("/memories?namespace=proj_a").json() - b_mems = client.get("/memories?namespace=proj_b").json() - - assert all(m["namespace"] == "proj_a" for m in a_mems) - assert all(m["namespace"] == "proj_b" for m in b_mems) - assert len(a_mems) == 1 - assert len(b_mems) == 1 - - -def test_namespace_isolates_search(): - client.post("/memories?namespace=proj_a", json={"content": "Retrieval test alpha."}) - client.post("/memories?namespace=proj_b", json={"content": "Retrieval test beta."}) - - results = client.get("/memories/search?q=retrieval+test&namespace=proj_a").json() - assert all(r["memory"]["namespace"] == "proj_a" for r in results) - - -def test_namespace_isolates_context(): - client.post("/memories?namespace=proj_a", json={"content": "Context alpha.", "importance": 0.9}) - client.post("/memories?namespace=proj_b", json={"content": "Context beta.", "importance": 0.9}) - - resp = client.get("/memories/context?namespace=proj_a").json() - assert all(m["namespace"] == "proj_a" for m in resp["memories"]) - - -def test_list_all_namespaces_when_omitted(): - client.post("/memories?namespace=proj_a", json={"content": "Alpha."}) - client.post("/memories?namespace=proj_b", json={"content": "Beta."}) - - all_mems = client.get("/memories").json() - namespaces = {m["namespace"] for m in all_mems} - assert "proj_a" in namespaces - assert "proj_b" in namespaces - - -def test_default_namespace_is_default(): - mid = client.post("/memories", json={"content": "Default NS memory."}).json()["id"] - mem = client.get(f"/memories/{mid}").json() - assert mem["namespace"] == "default" - - -# ── Snapshots: export / import ───────────────────────────────────────────── - - -def test_export_returns_all_memories(): - client.post("/memories", json={"content": "Export A."}) - client.post("/memories", json={"content": "Export B."}) - r = client.get("/memories/export") - assert r.status_code == 200 - data = r.json() - assert len(data) == 2 - assert all("id" in m and "content" in m for m in data) - - -def test_export_filtered_by_namespace(): - client.post("/memories?namespace=ns_x", json={"content": "NS X memory."}) - client.post("/memories?namespace=ns_y", json={"content": "NS Y memory."}) - data = client.get("/memories/export?namespace=ns_x").json() - assert len(data) == 1 - assert data[0]["namespace"] == "ns_x" - - -def test_export_empty_store(): - r = client.get("/memories/export") - assert r.status_code == 200 - assert r.json() == [] - - -def test_import_restores_memories(): - client.post("/memories", json={"content": "Original."}) - snapshot = client.get("/memories/export").json() - - # Clear and reimport - manager._store.clear() - r = client.post("/memories/import", json=snapshot) - assert r.status_code == 200 - result = r.json() - assert result["imported"] == 1 - assert result["skipped"] == 0 - assert result["total_in_snapshot"] == 1 - assert len(client.get("/memories").json()) == 1 - - -def test_import_skip_existing(): - mid = client.post("/memories", json={"content": "Already here."}).json()["id"] - snapshot = client.get("/memories/export").json() - - # Reimport with skip_existing=true (default) — same ID already present - r = client.post("/memories/import", json=snapshot) - result = r.json() - assert result["skipped"] == 1 - assert result["imported"] == 0 - assert len(client.get("/memories").json()) == 1 - - -def test_import_override_namespace(): - client.post("/memories?namespace=original", json={"content": "Move me."}) - snapshot = client.get("/memories/export").json() - - manager._store.clear() - client.post("/memories/import?namespace=overridden&skip_existing=false", json=snapshot) - mems = client.get("/memories").json() - assert all(m["namespace"] == "overridden" for m in mems) - - -def test_roundtrip_preserves_fields(): - client.post( - "/memories?namespace=snap_ns", - json={ - "content": "Full entry.", - "importance": 0.77, - "tags": ["a", "b"], - "memory_type": "semantic", - }, - ) - snapshot = client.get("/memories/export").json() - manager._store.clear() - client.post("/memories/import?skip_existing=false", json=snapshot) - - mems = client.get("/memories").json() - assert len(mems) == 1 - m = mems[0] - assert m["importance"] == pytest.approx(0.77) - assert m["tags"] == ["a", "b"] - assert m["namespace"] == "snap_ns" - assert m["memory_type"] == "semantic"