diff --git a/.claude/hooks/CLAUDE.md b/.claude/hooks/CLAUDE.md index aa388c6..9b7b174 100644 --- a/.claude/hooks/CLAUDE.md +++ b/.claude/hooks/CLAUDE.md @@ -64,3 +64,26 @@ neither mypy nor ruff could have found the original bug *as they were then configured*: both read the file at the project's own target version, under which it was valid. Pinning the floor in `ruff.toml` is what changes that, and the parse test is what proves it. + +**A fourth standing rule, added 2026-09-08 (failure-mode audit): +`--extend-ignore F401` on this hook's own `ruff check --fix` call.** +F401 (unused import) is in `pyproject.toml`'s `[tool.ruff.lint]` select, +so left unrestricted this hook silently deleted an import the instant +after it was added, whenever its first usage landed in a later Edit/ +Write call rather than the same one -- a routine sequence for an +agent's own edit-by-edit workflow, not a mistake. The deletion happened +between edits, so the failure it caused (`NameError`/F821 once the +usage landed) looked unrelated to this hook, and cost a re-add cycle +more than once before anyone connected the two -- see +`tests/integration/test_claude_hooks.py:: +test_hook_does_not_strip_an_import_with_no_usage_yet`, the regression +test that pins this. **This only defers F401 from "every edit" to +"commit time", it does not disable it**: `.pre-commit-config.yaml`'s own +`ruff` hook (`make lint`, part of `make ci`) runs with no rule +restriction, so a genuinely-unused import is still caught and fixed +before it merges. Root `CLAUDE.md`'s Tooling Gotchas section used to ask +a session to remember to add an import and its usage in the same edit +to work around exactly this by hand; that section now says the hook +itself no longer strips a not-yet-used import, and keeps the "re-read +after an import edit" advice as the belt-and-braces second line of +defence, not the only one. diff --git a/.claude/hooks/post_edit_format.py b/.claude/hooks/post_edit_format.py index 3d7a94a..76bc786 100644 --- a/.claude/hooks/post_edit_format.py +++ b/.claude/hooks/post_edit_format.py @@ -42,7 +42,23 @@ def main() -> int: if not target.exists(): return 0 - for args in (["ruff", "check", "--fix", str(target)], ["ruff", "format", str(target)]): + for args in ( + # --extend-ignore F401: this hook fires after every single Edit/ + # Write, but Claude Code's own workflow routinely adds an import in + # one call and its first usage in the next -- the file genuinely + # has an "unused" import for the instant between the two. Left + # unrestricted, `ruff check --fix` (F401 is in `pyproject.toml`'s + # `[tool.ruff.lint]` select) silently deletes it before the second + # edit lands, which then fails with NameError/F821 for a reason + # that looks unrelated to this hook. `make lint`/CI still catch a + # genuinely unused import at commit time -- `.pre-commit- + # config.yaml`'s `ruff` hook runs unrestricted -- so this only + # defers that one rule from "every edit" to "commit time", it does + # not disable it. See tests/integration/test_claude_hooks.py:: + # test_hook_does_not_strip_an_import_with_no_usage_yet. + ["ruff", "check", "--fix", "--extend-ignore", "F401", str(target)], + ["ruff", "format", str(target)], + ): result = subprocess.run( ["uv", "run", *args], cwd=REPO_ROOT, diff --git a/CLAUDE.md b/CLAUDE.md index c162ecd..c3770df 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -282,6 +282,12 @@ prevent (P-011, single authoritative source). ("targeting December 2026") -- the check matches only `YYYY-MM-DD`, which this repository uses exclusively for things that have already happened. +- `make check-duplicate-blocks` -- fail if a tracked Markdown file + contains the same large (12-line) block of prose twice, verbatim. + Added 2026-09-08 (failure-mode audit), for the structural shape a + botched `sed`/index-arithmetic edit leaves behind when it duplicates a + section instead of moving it -- see + `tools/validators/check_duplicate_blocks.py`'s own module docstring. - `make check-claims` -- report documentation claiming some file or directory is empty, unwritten, or a stub when it actually has content (`docs/practices.md`). **Advisory and deliberately outside `make ci`**: @@ -323,13 +329,16 @@ prevent (P-011, single authoritative source). - `make ci` -- `lint typecheck test check-docs check-docs-index check-graph check-dependency-tree check-inventory check-manifest check-references check-scenarios check-stages check-documents - check-status check-config-template check-dates check-benchmark-report` + check-status check-config-template check-dates check-duplicate-blocks + check-benchmark-report` together (this list itself went stale by two targets, `check-references` and - `check-scenarios`, before one correction, and by a third, + `check-scenarios`, before one correction, by a third, `check-benchmark-report`, added 2026-09-06 in the same change that - added the target -- restated facts drift even in the document that - warns about restated facts); this is what CI + added the target, and by a fourth, `check-duplicate-blocks`, added + 2026-09-08 in the same change that added it too -- restated facts + drift even in the document that warns about restated facts, and + keeps proving it every time a target is added here); this is what CI actually runs (`.github/workflows/ci.yml`), so it is also the one command that verifies a change is ready before committing. **For documentation it verifies structure, not content** diff --git a/Makefile b/Makefile index 3e3fc41..82739dc 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: install lint format typecheck test check-docs check-docs-index check-graph \ dependency-tree check-dependency-tree inventory check-inventory \ check-manifest check-references check-scenarios check-stages check-documents \ - check-claims check-dates status-report \ + check-claims check-dates check-duplicate-blocks status-report \ check-status config-template check-config-template docs graph demo benchmark \ benchmark-report check-benchmark-report record-benchmarks ci clean @@ -136,7 +136,7 @@ check-inventory: check-manifest: uv run python tools/validators/check_manifest.py -ci: lint typecheck test check-docs check-docs-index check-graph check-dependency-tree check-inventory check-manifest check-references check-scenarios check-stages check-documents check-status check-config-template check-dates check-benchmark-report +ci: lint typecheck test check-docs check-docs-index check-graph check-dependency-tree check-inventory check-manifest check-references check-scenarios check-stages check-documents check-status check-config-template check-dates check-duplicate-blocks check-benchmark-report # Fails if prose names a repository path that does not exist. Gating: # every rule is a definite structural fact (does this path resolve), @@ -192,6 +192,17 @@ check-documents: check-dates: uv run python tools/validators/check_dates.py +# Fails if a tracked Markdown file contains the same large (12-line) +# block of prose twice, verbatim -- the structural shape a botched +# sed/index-based reorder leaves behind (a real ~3,900-line incident, +# never itself committed since it was caught and reverted within a +# session -- see tools/validators/check_duplicate_blocks.py's own module +# docstring). Gating, added 2026-09-08 (failure-mode audit): whether a +# specific run of lines repeats verbatim elsewhere in the same file is a +# structural fact, not a judgement call. +check-duplicate-blocks: + uv run python tools/validators/check_duplicate_blocks.py + check-claims: uv run python tools/validators/check_claims.py diff --git a/docs/planning/roadmap.md b/docs/planning/roadmap.md index 2dfcf25..9160c1b 100644 --- a/docs/planning/roadmap.md +++ b/docs/planning/roadmap.md @@ -306,7 +306,11 @@ This paragraph previously said `make install` and `make test` were still expected to fail, pending `uv.lock` and a test suite (B2/C1) -- stale since 2026-08-16 and corrected 2026-08-19. Both now succeed: `uv.lock` is committed (B2) and `make test` runs the suite with coverage -(C1a/C1b): **1163 tests as of 2026-09-08**, up from 1160 the day before +(C1a/C1b): **1171 tests as of 2026-09-08**, up from 1163 the same day (8 +new tests from the failure-mode audit: `test_hook_does_not_strip_an_ +import_with_no_usage_yet`, two `check_manifest.py` tests for the new +`claude-md-count-matches-live` rule, and five for the new +`check_duplicate_blocks.py`), 1163 itself up from 1160 the day before (three new `check_manifest.py` tests, `ka-name-matches-manifest`), then 1154 slightly earlier that day (below), then 1153, 1143, 1137, 1131, and 1052 the day before that. diff --git a/docs/planning/status.md b/docs/planning/status.md index 38ad29b..a266e9d 100644 --- a/docs/planning/status.md +++ b/docs/planning/status.md @@ -46,7 +46,7 @@ pie showData ## Live repository facts - **47** `CLAUDE.md` files -- **1163** tests collected +- **1171** tests collected - **144** Gherkin scenarios (`tests/features/*.feature`) ## Stages diff --git a/docs/repository-inventory.md b/docs/repository-inventory.md index 80ad20c..725de2f 100644 --- a/docs/repository-inventory.md +++ b/docs/repository-inventory.md @@ -16,7 +16,7 @@ reading job and lives in the manifest. Test counts and coverage are not here either -- those come from running the suite, not from listing files. -**358 tracked files** across 47 directories; +**360 tracked files** across 47 directories; 2 are empty. ## (root) @@ -414,6 +414,7 @@ listing files. - `test_check_dates.py` - `test_check_docs.py` - `test_check_documents.py` +- `test_check_duplicate_blocks.py` - `test_check_graph.py` - `test_check_manifest.py` - `test_check_references.py` @@ -512,6 +513,7 @@ listing files. - `check_dates.py` - `check_docs.py` - `check_documents.py` +- `check_duplicate_blocks.py` - `check_graph.py` - `check_manifest.py` - `check_references.py` diff --git a/docs/repository-manifest.md b/docs/repository-manifest.md index 8d6ef80..4f062bc 100644 --- a/docs/repository-manifest.md +++ b/docs/repository-manifest.md @@ -1154,12 +1154,15 @@ manifest or covered by one of its collective rules, gating, 2026-08-21), and dropped, see `tools/validators/CLAUDE.md`) `check_scenarios.py` (a Gherkin scenario nothing binds, gating, 2026-08-22), `check_stages.py` (a Stage missing part of the shape `docs/planning/stage-shape.yaml` -declares, gating, 2026-09-03) and `check_documents.py` (a maintained +declares, gating, 2026-09-03), `check_documents.py` (a maintained document not declaring what keeps it honest -- generated, gated, or stage-boundary re-read -- gating, 2026-09-03; it also prints the list of documents nothing checks mechanically, which is the reading list an exit audit needs and which is therefore derived rather than restated -anywhere); +anywhere), and `check_duplicate_blocks.py` (the same large, mostly- +substantial block of Markdown prose appearing twice verbatim in one +file -- the shape a botched sed/index-based reorder leaves behind, +gating, 2026-09-08, failure-mode audit); `generators/` holds `generate_dependency_tree.py` (`docs/planning/dependency-tree.md` from the component graph, 2026-08-21), `generate_repository_inventory.py` @@ -1267,38 +1270,34 @@ They are tracked collectively here, not as individual rows, because per-directory agent guidance is a property of the directory rather than a standalone artifact (KA-038). -As of 2026-08-23: **45 files exist; 4 are still the generic placeholder** -and 41 carry real local content. (Read "42 ... and 38", as of -2026-08-22, until 2026-08-23 -- three files joined in between and this -count was not updated for any of them: `tests/features/CLAUDE.md` -(added by the same change as ADR-007, 2026-08-22, real content, missed -by the very consistency sweep that landed hours earlier) and -`src/pyflow/engine/numerics/CLAUDE.md`/`tests/unit/numerics/CLAUDE.md` -(TASK-018, 2026-08-23, both real content). Found while drafting this -same TASK-018 change, the same "count restated in three places, one -file added, count not touched" failure this row already exists to warn -about.) (42 rather than 40 because F2 -(`docs/planning/backlog.md`) found `.claude/` and `.claude/hooks/` -untracked by this manifest and by `docs/planning/knowledge-architecture.md`, -with no `CLAUDE.md` at all -- both written in the same change, both real -content, not placeholders. 40 itself down from 43 because `assets/icons/`, -`assets/shaders/`, `assets/textures/` were retired 2026-08-19, E9, taking -their placeholder files with them, on the same "nothing states what this -is for" test that retired `tools/planner/`/`tools/scripts/`, E10; 43 -itself down from 45 for that same E10 retirement.) E9's *Done when* was -revised the same day it closed: no placeholder may remain in a directory -that has content, not no placeholder anywhere -- inventing -directory-specific guidance for a directory that is still genuinely -empty produces speculation, not knowledge. **3** remaining placeholders -as of 2026-09-04 (`docs/tutorials/`, `examples/tutorials/`, -`tests/performance/`) sit in directories with no real content yet -- -down from 4, `examples/experiments/` having gained real content that -day (`smoke_transport_high_res.yaml`, a higher-resolution variant of -`examples/golden-demos/smoke_transport.yaml`) -- so E9 is closed under -the revised criterion. `docs/planning/backlog.md` E9 -holds the file-by-file breakdown and is the authoritative count; this -row and `docs/planning/roadmap.md`'s TASK-009 status both restate it, so -update all three together. `examples/experiments/` gained a second file +As of 2026-09-08: **47 files exist; 3 are still the generic placeholder** +and 44 carry real local content. **This row had drifted to 2026-08-23's +count of 45 while `docs/planning/roadmap.md`'s TASK-009 status kept +being updated to 46 and then 47** -- found and corrected in this same +change by the new `claude-md-count-matches-live` rule +(`tools/validators/check_manifest.py`, `make check-manifest`, added in +this same change), which now cross-checks this exact claim against the +live count on every run rather than leaving it to the next person who +happens to compare the two by hand. `docs/planning/roadmap.md`'s +TASK-009 row carries the full incremental history (47 from +`tools/benchmarks/CLAUDE.md`; 46 from `tests/fixtures/CLAUDE.md`, +TASK-034; 45 from `tests/features/CLAUDE.md`, ADR-007, plus +`src/pyflow/engine/numerics/CLAUDE.md`/`tests/unit/numerics/CLAUDE.md`, +TASK-018; 42 from F2 finding `.claude/` and `.claude/hooks/` untracked; +40 from retiring `assets/icons/`/`assets/shaders/`/`assets/textures/`, +E9/E10) -- this row restates only the current total rather than +re-deriving that whole chain a second time, since restating it fully in +both places is exactly the duplication that let this row go stale for +two updates running. `docs/planning/backlog.md` E9 holds the +file-by-file breakdown as of its own 2026-08-19 closure and is not kept +current past that point -- it is a record of when E9 closed, not a +running total; this row and `docs/planning/roadmap.md`'s TASK-009 status +are the two that track the live count, so update both together. **3** +files sit in directories with no real content yet (`docs/tutorials/`, +`examples/tutorials/`, `tests/performance/`), unchanged since 2026-09-04 +(`docs/planning/backlog.md` E9's revised *Done when*: no placeholder may +remain in a directory that has content, not no placeholder anywhere). +`examples/experiments/` gained a second file 2026-09-06, `smoke_transport_re1000.yaml` -- a 64x64, Re = 1000 trial of the same smoke-transport shape, checking whether more mesh and a higher Reynolds number make secondary corner vortices visible where Re = 100 diff --git a/tests/integration/test_claude_hooks.py b/tests/integration/test_claude_hooks.py index a20edaf..3556dc9 100644 --- a/tests/integration/test_claude_hooks.py +++ b/tests/integration/test_claude_hooks.py @@ -84,6 +84,56 @@ def test_settings_json_wires_up_at_least_one_hook() -> None: assert _configured_hook_commands() +def _post_edit_format_command() -> str: + """The one configured hook command whose target is `post_edit_format.py`. + + Read from `.claude/settings.json` rather than hardcoded, same as + `_configured_hook_commands` above, so a change to how it is invoked is + picked up automatically rather than silently going untested. + """ + for command in _configured_hook_commands(): + if "post_edit_format.py" in command: + return command + raise AssertionError("no configured hook command targets post_edit_format.py") + + +def test_hook_does_not_strip_an_import_with_no_usage_yet(tmp_path: Path) -> None: + """A newly-added import with no usage yet must survive the hook. + + Regression test (2026-09-08, failure-mode audit). `ruff check --fix`'s + configured rule set (`pyproject.toml`'s `[tool.ruff.lint]`, `"F"`) + includes F401 (unused import), and an Edit/Write-by-edit workflow + routinely adds an import in one call and its first usage in the next -- + the file genuinely has an unused import for the instant between the + two. Without a carve-out this hook fires after the first edit and + silently deletes the import before the second edit lands, which then + fails with `NameError`/F821 for a reason that looks unrelated to the + hook that caused it. Confirmed as a real, repeated cost before this + test existed (Claude Code Insights, 2026-09-08 usage report): a + formatter hook silently dropping a just-added `import math` cost a + re-add cycle across multiple sessions, never itself visible in `git + log` because the workaround happened inside a session, before anything + was committed. + """ + target = tmp_path / "not_yet_used.py" + target.write_text("import math\n", encoding="utf-8") + payload = json.dumps({"tool_input": {"file_path": str(target)}}) + + result = subprocess.run( + shlex.split(_post_edit_format_command()), + input=payload, + capture_output=True, + text=True, + cwd=REPO_ROOT, + check=False, + ) + + assert result.returncode == 0, f"hook failed: {result.stderr}" + assert "import math" in target.read_text(encoding="utf-8"), ( + "the hook stripped an import that simply has no usage yet" + ) + + @pytest.mark.parametrize("command", _configured_hook_commands()) def test_configured_hook_runs_and_formats_the_file_it_is_given( command: str, tmp_path: Path diff --git a/tests/unit/test_check_duplicate_blocks.py b/tests/unit/test_check_duplicate_blocks.py new file mode 100644 index 0000000..241ee23 --- /dev/null +++ b/tests/unit/test_check_duplicate_blocks.py @@ -0,0 +1,96 @@ +"""Unit tests for tools/validators/check_duplicate_blocks.py. + +Fixture repos in `tmp_path`, same reasoning as `test_check_manifest.py`: +the real repository is checked once, at the bottom, and everything else +builds a miniature repository so a rule's own test doesn't fail whenever +real prose legitimately changes. +""" + +import subprocess +import sys +from pathlib import Path + +TOOLS_VALIDATORS = Path(__file__).resolve().parents[2] / "tools" / "validators" +if str(TOOLS_VALIDATORS) not in sys.path: + sys.path.insert(0, str(TOOLS_VALIDATORS)) + +from check_duplicate_blocks import check_duplicate_blocks # noqa: E402 + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _repo(tmp_path: Path, files: dict[str, str]) -> Path: + """A miniature git repository holding the given tracked files. + + A real `git init`/`git add`, because this validator asks git which + Markdown files are tracked rather than walking the disk. + """ + for name, body in files.items(): + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(body, encoding="utf-8") + for command in (["init", "-q"], ["add", "-A"]): + subprocess.run(["git", *command], cwd=tmp_path, capture_output=True, check=True) + return tmp_path + + +def _substantial_block(n: int, prefix: str) -> str: + """`n` lines that are each long enough to count as "substantial".""" + return "\n".join(f"{prefix} line number {i} carries real, specific prose" for i in range(n)) + + +def test_a_large_duplicated_block_is_reported(tmp_path: Path) -> None: + """The failure this exists for, at a testable scale: a botched sed or + index-based reorder duplicated roughly 3,900 lines of a real planning + document (Claude Code Insights, 2026-09-08 usage report) -- the same + substantial block of prose appearing twice, verbatim, in one file. + """ + block = _substantial_block(15, "content") + text = "intro\n\n" + block + "\n\nan unrelated middle section\n\n" + block + "\n\noutro\n" + root = _repo(tmp_path, {"docs/plan.md": text}) + + findings = check_duplicate_blocks(root) + assert len(findings) == 1 + assert "docs/plan.md" in findings[0] + + +def test_short_or_sparse_repeated_lines_are_not_reported(tmp_path: Path) -> None: + """Table dividers, blank-line runs and short list markers repeat + legitimately throughout real documents and must never fire this + check -- only a run of genuinely substantial lines should count. + """ + text = "\n".join(["| a | b |", "|---|---|"] * 20) + root = _repo(tmp_path, {"docs/table.md": text}) + + assert check_duplicate_blocks(root) == [] + + +def test_a_file_with_no_duplication_is_not_reported(tmp_path: Path) -> None: + block = _substantial_block(15, "content") + root = _repo(tmp_path, {"docs/plan.md": "intro\n\n" + block + "\n\noutro\n"}) + + assert check_duplicate_blocks(root) == [] + + +def test_a_non_markdown_tracked_file_is_not_scanned(tmp_path: Path) -> None: + """Scoped to `*.md` only (root CLAUDE.md's Blast Radius rule is about + documentation specifically here) -- code and tests routinely repeat + near-identical structure on purpose (parametrised tests, per-field + config comments) and scanning them would reproduce the false-positive + trap `check_manifest.py`'s own dropped "every path exists" rule + warns about. + """ + block = _substantial_block(15, "content") + text = block + "\n\n" + block + "\n" + root = _repo(tmp_path, {"src/generated.py": text}) + + assert check_duplicate_blocks(root) == [] + + +def test_the_real_repositorys_markdown_has_no_duplicated_blocks() -> None: + """Proves the rule against real data before it gates anything -- the + same discipline `check_manifest.py`'s `ka-name-matches-manifest` was + verified with: run against the real repository and confirm zero + findings before wiring it into `make ci`. + """ + assert check_duplicate_blocks(REPO_ROOT) == [] diff --git a/tests/unit/test_check_manifest.py b/tests/unit/test_check_manifest.py index 60ba1ad..25fbffd 100644 --- a/tests/unit/test_check_manifest.py +++ b/tests/unit/test_check_manifest.py @@ -239,6 +239,51 @@ def test_a_manifest_row_citing_an_unknown_ka_is_not_reported_by_this_rule(tmp_pa assert check_manifest(root) == [] +def test_manifest_claude_md_count_disagreeing_with_live_count_is_reported( + tmp_path: Path, +) -> None: + """The exact drift a 2026-09-08 failure-mode audit found by hand: + `generate_status_report.py`'s `check-status` cross-checks + `docs/planning/roadmap.md`'s own "N files exist" claim against the + live count, but nothing checked this document's *own*, separate + restatement of the same fact -- and it had drifted two updates behind + while roadmap.md kept being updated, unnoticed until this rule was + written specifically because that drift had already happened once. + """ + root = _repo( + tmp_path, + manifest=( + "# Repository Manifest\n\n" + "# CLAUDE.md files\n\n" + "As of 2026-08-23: **1 files exist**; 0 are still placeholders.\n\n" + "- `README.md`\n" + ), + files={"README.md": "hi\n", "src/CLAUDE.md": "x\n", "docs/CLAUDE.md": "y\n"}, + ) + + findings = _findings(root) + assert "claude-md-count-matches-live" in findings + assert "claims 1" in findings + assert "2 exist" in findings + + +def test_manifest_claude_md_count_agreeing_with_live_count_is_not_reported( + tmp_path: Path, +) -> None: + root = _repo( + tmp_path, + manifest=( + "# Repository Manifest\n\n" + "# CLAUDE.md files\n\n" + "As of 2026-08-23: **2 files exist**; 0 are still placeholders.\n\n" + "- `README.md`\n" + ), + files={"README.md": "hi\n", "src/CLAUDE.md": "x\n", "docs/CLAUDE.md": "y\n"}, + ) + + assert check_manifest(root) == [] + + def test_the_repositorys_own_manifest_passes() -> None: """A different assertion from every test above: those prove the rules fire, this proves the real manifest satisfies them. Named so a diff --git a/tools/validators/CLAUDE.md b/tools/validators/CLAUDE.md index 90ab0ee..5d4efca 100644 --- a/tools/validators/CLAUDE.md +++ b/tools/validators/CLAUDE.md @@ -296,6 +296,33 @@ impossible -- work dated three days early, say. `docs/practices.md`'s end-of-session review step 11c is that half, and it says to compare against `git log`. +**`check_duplicate_blocks.py`** (added 2026-09-08, failure-mode audit) +fails if a tracked Markdown file contains the same large (12-line, +mostly-substantial) block of prose twice, verbatim. Mechanises a failure +mode nothing else here covered: a `sed`/index-arithmetic edit that +duplicates a section of a file instead of moving it -- a real incident +duplicated roughly 3,900 lines of a planning document this way (Claude +Code Insights, 2026-09-08 usage report). **That incident never reached +`git log`**, because it was caught and reverted within the session +before anything was committed -- unlike every other validator in this +file, which points at a specific commit or CI run its own rule was built +from, this one is built from the *shape* of a failure with no commit to +cite, the same way `check_dates.py`'s escape hatch was designed from a +class of mistake rather than a single instance. + +**Scoped to `*.md` only, deliberately.** Code and tests repeat +near-identical structure on purpose (parametrised tests, per-field +config comments); scanning them would very likely reproduce the +false-positive trap `check_manifest.py`'s own dropped "every path +exists" rule warns about, two entries below. The window size and +substantiality threshold were tuned the same way `ka-name-matches- +manifest` was verified: run against this repository's real tracked +Markdown and confirm zero findings before landing, not merely assumed +safe. Proven to fire, then reverted, against a real duplicated chunk of +`docs/planning/backlog.md` before this validator was wired into +`make ci` -- see the module's own docstring for the exact thresholds and +why each was chosen. + **`check_manifest.py`** (added 2026-08-21) enforces the contract `docs/repository-manifest.md` states about itself: "Every maintained file should appear here exactly once, either as its own row or under an @@ -348,3 +375,20 @@ id repeats, every manifest citation resolves to a real KA heading, and every citing row already agreed with its KA entry's `**Name:**` field -- the gate landed with zero findings against `docs/repository-manifest.md` and `docs/planning/knowledge-architecture.md` as they stood that day. + +**`claude-md-count-matches-live`, added 2026-09-08 (failure-mode +audit).** This manifest's own "CLAUDE.md files" section states "As of +DATE: **N files exist**"; this rule checks N against the live count of +tracked files named `CLAUDE.md`. It exists because that exact drift had +already happened: `generate_status_report.py`'s `check-status` (`tools/ +generators/CLAUDE.md`) cross-checks `docs/planning/roadmap.md`'s own +copy of this count against reality, but this manifest keeps a *second*, +independent restatement of the same fact, and nothing checked it. It sat +two updates behind the roadmap's copy -- found only when this rule was +being written and run against the real manifest for the first time, the +same way `ka-name-matches-manifest` above found its own zero-finding +baseline by actually running against real data rather than assuming it. +Deliberately as narrow as that rule: only this one claim, phrased in one +fixed form, is extracted and checked -- not a general "every number in +this document is checked" rule, which would need a reader the same way +`check_claims.py`'s advisory scope already explains. diff --git a/tools/validators/check_duplicate_blocks.py b/tools/validators/check_duplicate_blocks.py new file mode 100644 index 0000000..0037de3 --- /dev/null +++ b/tools/validators/check_duplicate_blocks.py @@ -0,0 +1,127 @@ +"""Fail if a tracked Markdown file contains the same large block of prose +twice, verbatim. + +Mechanises the failure mode a 2026-09-08 audit of this repository's own +recurring mistakes found no other gate covered: a `sed`/index-arithmetic +edit that duplicates a section of a file instead of moving it, which +duplicated roughly 3,900 lines of a real planning document in one +incident (Claude Code Insights, 2026-09-08 usage report). That incident +never reached `git log` -- it was caught and reverted within the session +before being committed -- so there is no historical commit this script +can point at; it is built from the failure's shape, not a specific +commit, the same way `check_dates.py` was built from a class of mistake +rather than one instance of it. + +**Detection: a sliding window of `WINDOW` consecutive lines, hashed as a +tuple.** If the same `WINDOW`-line block appears twice in one file, at +non-overlapping offsets, that is reported. A window is only compared once +at least `MIN_SUBSTANTIAL_LINES` of its `WINDOW` lines are "substantial" +(at least `MIN_LINE_LENGTH` characters after stripping) -- table +dividers, blank-line runs and short list markers repeat legitimately +throughout real prose and must not fire this on their own. Both +constants were chosen empirically against this repository's own tracked +Markdown (verified zero findings before landing, the same discipline +`tools/validators/CLAUDE.md`'s `ka-name-matches-manifest` entry +describes) -- widen `WINDOW` or lower `MIN_SUBSTANTIAL_LINES` only after +re-running against the real tree and confirming it still finds nothing. + +**Scoped to `*.md` only.** Code and tests routinely repeat near-identical +structure on purpose (parametrised tests, per-field config comments, +boilerplate CLAUDE.md headers) -- scanning them would very likely +reproduce the false-positive trap `check_manifest.py`'s own dropped +"every path exists" rule warns about (`tools/validators/CLAUDE.md`). +Prose duplication is also the actual incident this exists for: a +structural copy-paste accident in a *document*, not a legitimate +repeated code pattern. + +Run via `make check-duplicate-blocks`, part of `make ci`. Gates rather +than advises: whether a specific run of lines repeats verbatim elsewhere +in the same file is a structural fact, not a judgement call, the same +reasoning `check_graph.py`/`check_manifest.py` already use. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] + +# See the module docstring for how these were chosen. WINDOW=12 with +# MIN_SUBSTANTIAL_LINES=10 is far below the ~3,900-line incident this +# check exists to catch, while producing zero findings against this +# repository's own tracked Markdown as of 2026-09-08. +WINDOW = 12 +MIN_SUBSTANTIAL_LINES = 10 +MIN_LINE_LENGTH = 20 + + +def _tracked_markdown_files(root: Path) -> list[Path]: + result = subprocess.run( + ["git", "ls-files", "*.md"], cwd=root, capture_output=True, text=True, check=True + ) + return [root / p for p in result.stdout.split()] + + +def _is_substantial(block: tuple[str, ...]) -> bool: + substantial = sum(1 for line in block if len(line.strip()) >= MIN_LINE_LENGTH) + return substantial >= MIN_SUBSTANTIAL_LINES + + +def _find_duplicate(lines: list[str]) -> tuple[int, int] | None: + """First (earlier_index, later_index) pair of a duplicated, non- + overlapping `WINDOW`-line block, or `None`. + """ + seen: dict[tuple[str, ...], int] = {} + for index in range(len(lines) - WINDOW + 1): + block = tuple(lines[index : index + WINDOW]) + if not _is_substantial(block): + continue + first = seen.get(block) + if first is None: + seen[block] = index + elif index >= first + WINDOW: + return first, index + return None + + +def check_duplicate_blocks(root: Path = REPO_ROOT) -> list[str]: + """Every duplicated block found, as human-readable strings. + + `root` is a parameter so tests can build miniature repositories in + `tmp_path` -- a test asserting only against real files would fail + every time real prose legitimately changed near a false positive. + """ + findings: list[str] = [] + for path in _tracked_markdown_files(root): + if not path.is_file(): + continue + lines = path.read_text(encoding="utf-8").splitlines() + result = _find_duplicate(lines) + if result is None: + continue + first, second = result + rel = path.relative_to(root).as_posix() + findings.append( + f"duplicate-block: {rel} repeats the same {WINDOW}-line block at " + f"line {first + 1} and line {second + 1} -- check for a botched " + f"sed/index-based edit rather than an intentional move" + ) + return findings + + +def main() -> int: + findings = check_duplicate_blocks() + if findings: + for finding in findings: + print(finding) + print(f"\n{len(findings)} duplicate block(s) found.") + return 1 + + print("No duplicated content blocks found.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/validators/check_manifest.py b/tools/validators/check_manifest.py index 0e2fa4a..25749be 100644 --- a/tools/validators/check_manifest.py +++ b/tools/validators/check_manifest.py @@ -39,6 +39,20 @@ likely reproduce that rule's own false-positive failure. Scoping to "both sides already agree the id exists" keeps every finding a structural fact instead. +- claude-md-count-matches-live: this document's own "CLAUDE.md files" + section states "As of DATE: **N files exist**"; N must equal the live + count of tracked files named `CLAUDE.md`. Added 2026-09-08 + (failure-mode audit) after finding, by hand, that this exact claim had + drifted two updates behind `docs/planning/roadmap.md`'s own copy of the + same fact -- `tools/generators/generate_status_report.py`'s + `check-status` already cross-checks the roadmap's copy against the live + count, but nothing checked this document's *separate* restatement of + it, so the two could (and did) disagree silently. Deliberately narrow, + the same shape as `ka-name-matches-manifest` above: this is the one + claim in this document phrased consistently enough (a fixed "As of + DATE: **N files exist**" form) to extract and check by regex without + needing a reader, not a general "every count in this document is + checked" rule. **A fourth rule was built and removed rather than shipped** (2026-08-21): "every path the manifest names exists". It produced 44 findings on the @@ -88,6 +102,9 @@ KA_HEADING = re.compile(r"^## (KA-\d{3})\b", re.MULTILINE) KA_NAME = re.compile(r"\*\*Name:\*\*\s*~{0,2}`([^`]+)`") KA_CITATION = re.compile(r"\(KA-(\d{3})\)") +# The manifest's own "CLAUDE.md files" section states this in a fixed form +# -- see the RULES docstring's claude-md-count-matches-live entry. +MANIFEST_CLAUDE_MD_CLAIM = re.compile(r"As of \d{4}-\d{2}-\d{2}:\s*\*\*(\d+)\s+files exist") def _ka_names(ka_doc: str) -> dict[str, str]: @@ -203,6 +220,17 @@ def check_manifest(root: Path = REPO_ROOT) -> list[str]: f"names {row_name}" ) + # -- claude-md-count-matches-live ------------------------------------ + claim = MANIFEST_CLAUDE_MD_CLAIM.search(manifest) + if claim is not None: + claimed = int(claim.group(1)) + live = sum(1 for path in tracked if Path(path).name == "CLAUDE.md") + if claimed != live: + findings.append( + f"claude-md-count-matches-live: manifest claims {claimed} CLAUDE.md " + f"files, but {live} exist in the repository (`git ls-files`)" + ) + return findings