From 58e0191b6985fc3aeb256f2b4af0662b5577bc10 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Mon, 24 Aug 2026 22:05:59 +0000 Subject: [PATCH 1/5] fix(check): scope Edit fragment unreliability to the actual edge An Edit/MultiEdit fragment marked every newly-added comment's context unreliable, so classify's catch-all rewrote the verdict to Justified and the hook passed. Restatement and flow-narration detection were dead on the tool agents use to modify existing files: only AgentMemo, CommentedOutCode and VacuousTodo could ever fire. A fragment is contiguous file text, so a comment with code after it inside the fragment annotates that code in the file too. Mark unreliable only where the boundary could have taken the adjacent code away: no adjacent code at all, or nothing after the comment. This restores the edge-scoped rule the U2 plan specifies (plan lines 233, 248). Gate: fmt, clippy -D warnings, 15 pipeline tests, F1 unchanged. --- crates/comment-checker/src/check.rs | 11 +++------ crates/comment-checker/tests/pipeline.rs | 31 ++++++++++++------------ 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/crates/comment-checker/src/check.rs b/crates/comment-checker/src/check.rs index 225ac62..488205a 100644 --- a/crates/comment-checker/src/check.rs +++ b/crates/comment-checker/src/check.rs @@ -2,7 +2,7 @@ use crate::Verdict; use crate::classify::classify; -use crate::comment::Comment; +use crate::comment::{Comment, PositionRole}; use crate::detect::detect_comments; use crate::hook::{HookInput, decode}; use crate::report::{Flagged, format_report}; @@ -67,16 +67,13 @@ fn new_comments(old: &str, new: &str, file_path: &str) -> Vec { detect_comments(new, file_path) .into_iter() .filter(|comment| !old_texts.contains(&normalize(comment))) - .map(mark_unreliable) + .map(mark_fragment_edge_context) .collect() } -/// Mark a comment's context as unreliable — the fragment edge of an Edit or -/// `MultiEdit` may have lost adjacent-code context, so the verifier must -/// fall back from restate detection rather than convict. -fn mark_unreliable(mut comment: Comment) -> Comment { +fn mark_fragment_edge_context(mut comment: Comment) -> Comment { if let Some(ctx) = comment.context.as_mut() { - ctx.unreliable = true; + ctx.unreliable = ctx.adjacent_code.is_none() || ctx.position == PositionRole::Trailing; } comment } diff --git a/crates/comment-checker/tests/pipeline.rs b/crates/comment-checker/tests/pipeline.rs index 4cfc145..eff33d8 100644 --- a/crates/comment-checker/tests/pipeline.rs +++ b/crates/comment-checker/tests/pipeline.rs @@ -61,30 +61,31 @@ fn edit_new_comment_blocks() { } #[test] -fn edit_fragment_context_is_never_relied_upon() { - // U2 scenario: an Edit sees only the fragment, so a comment that would - // restate its adjacent code must NOT be convicted — the text-only floor - // downgrades, the hook passes, and the user is not blocked on context the - // fragment cannot vouch for. +fn edit_new_restatement_with_code_after_it_blocks() { let input = r#"{"tool_name":"Edit","tool_input":{"file_path":"foo.py","old_string":"counter = 0\n","new_string":"counter = 0\n# increment the counter\ncounter += 1\n"}}"#; - assert!( - matches!(check(input, ""), Outcome::Pass { .. }), - "an Edit fragment whose context is unreliable must fall back, not convict" - ); + assert!(matches!(check(input, ""), Outcome::Block { .. })); } #[test] -fn multi_edit_new_restatement_comment_passes() { - // MultiEdit fragments share Edit's unreliable-context rule: a would-be - // restatement introduced by an edit is spared, not convicted. - let input = r#"{"tool_name":"MultiEdit","tool_input":{"file_path":"foo.py","edits":[{"old_string":"x = 1\n","new_string":"x = 1\n# increment the counter\ncounter += 1\n"}]}}"#; +fn edit_comment_at_the_fragment_tail_passes() { + let input = r#"{"tool_name":"Edit","tool_input":{"file_path":"foo.py","old_string":"counter = 0\n","new_string":"counter = 0\n# increment the counter\n"}}"#; assert!(matches!(check(input, ""), Outcome::Pass { .. })); } +#[test] +fn edit_comment_with_no_adjacent_code_passes() { + let input = r##"{"tool_name":"Edit","tool_input":{"file_path":"foo.py","old_string":"","new_string":"# increment the counter\n"}}"##; + assert!(matches!(check(input, ""), Outcome::Pass { .. })); +} + +#[test] +fn multi_edit_new_restatement_comment_blocks() { + let input = r#"{"tool_name":"MultiEdit","tool_input":{"file_path":"foo.py","edits":[{"old_string":"x = 1\n","new_string":"x = 1\n# increment the counter\ncounter += 1\n"}]}}"#; + assert!(matches!(check(input, ""), Outcome::Block { .. })); +} + #[test] fn multi_edit_new_todo_comment_blocks() { - // Explicit text-only rules still block on MultiEdit: the unreliable - // downgrade only spares the restate fallback, never a real rule match. let input = r#"{"tool_name":"MultiEdit","tool_input":{"file_path":"foo.py","edits":[{"old_string":"x = 1\n","new_string":"x = 1\n# TODO: handle\n"}]}}"#; assert!(matches!(check(input, ""), Outcome::Block { .. })); } From 73dfa7dd5b08e20725eac98ec82a7951e3a5a9df Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Mon, 24 Aug 2026 22:14:35 +0000 Subject: [PATCH 2/5] docs(plans): plan the justification-laundering fix A keyword acquits a proven restatement: 13 of 18 paraphrases of one restatement are spared by adding a marker word. Conviction is evidence-backed and default-deny; acquittal is bare substring membership. The plan inverts that precedence, narrows attribution to lead-position tags, re-derives the corpus floors, and stops the terminal path claiming a restatement it never cited. Checked and NOT holes: is_bdd (exact equality), DIRECTIVE_PREFIXES (tool-specific tokens only). --- ...-08-24-001-fix-justification-laundering.md | 215 ++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 docs/plans/2026-08-24-001-fix-justification-laundering.md diff --git a/docs/plans/2026-08-24-001-fix-justification-laundering.md b/docs/plans/2026-08-24-001-fix-justification-laundering.md new file mode 100644 index 0000000..e7e116a --- /dev/null +++ b/docs/plans/2026-08-24-001-fix-justification-laundering.md @@ -0,0 +1,215 @@ +--- +artifact_contract: ce-unified-plan/v1 +artifact_readiness: implementation-ready +product_contract_source: ce-plan-bootstrap +execution: code +created: 2026-08-24 +updated: 2026-08-24 +type: fix +--- + +# Justification laundering: a keyword must not acquit a restatement + +## Goal Capsule + +- **Objective:** Appending or prepending a marker word to a comment that restates its adjacent code must not change the verdict. Today it does, for 13 of 18 paraphrases of one restatement. +- **Product authority:** User-directed. The reported failure is that the hook drives a 10-line comment block down to 2 lines instead of to zero — the surviving lines are the ones carrying a marker word. +- **Open blockers:** None. +- **Execution profile:** code. Characterize, then invert one precedence, then narrow one table, then re-derive the corpus floors. +- **Stop conditions:** The probe fixture's spare-rate is 0 for markers attached to a proven restatement, the F1 gate is green on re-derived labels, and `classify.rs` mutation stays 100%. + +--- + +## Product Contract + +### Summary + +Conviction in this classifier is evidence-backed and default-deny; acquittal is bare substring membership. That asymmetry is the whole defect. A comment that demonstrably restates its adjacent code is acquitted if it happens to contain `because`, `note:`, `security`, `1-based`, `ref:` or one of 30-odd other common English fragments. + +### Problem Frame + +Measured against a release build of `master` + the fragment-edge fix, `2026-08-24`, by piping `PostToolUse` payloads to `target/release/comment-checker` and reading the exit code (2 = block, 0 = spare): + +- 18 comments, each meaning exactly `set x to 1`, above `const x = 1;`. **13 spared, 5 blocked.** The 5 blocked are the ones carrying no marker. +- Padding is not the escape: `// set x to 1` plus 1, 3, 6, or 9 words of unrelated prose still blocks. Conviction does not depend on the containment ratio clearing a bar, because the catch-all convicts on empty evidence (`classify.rs` terminal rule). +- The escape is therefore entirely the `JUSTIFIED` table, and within it two rules: `is_non_obvious_intent` over `INTENT_MARKERS` (28 substrings incl. `why`, `because`, `must be`, `security`, `algorithm`, `regex`, `1-based`) and `is_attribution` over `ATTRIBUTION_MARKERS` + `ATTRIBUTION_PREFIXES` (incl. `based on`, `credit`, `adapted from`, `ref:`, `source:`). +- `is_bdd` is **not** a hole: `BDD_KEYWORDS.contains(&s)` is exact equality on the stripped text, so `// when the flag is on` blocks. `DIRECTIVE_PREFIXES` is **not** a hole: it carries only tool-specific tokens, so `// Allows the caller to…` blocks. Both were checked and both hold. +- Unsupported extensions pass unenforced (`// set x to 1` in `a.foo` spares). Known, by design, out of scope here. + +`JUSTIFIED` is consulted before `UNNECESSARY` and before the context-aware detectors (`classify.rs` `classify`), so a marker word wins before any evidence is computed. The precedence, not the table contents, is what makes the tool launderable. + +### Requirements + +- R1. A comment whose restatement of its adjacent code is provable with cited evidence is convicted, regardless of any marker it contains. +- R2. A marker that is a doc-tool tag (`@author`, `@copyright`, `@see`, `@link`) justifies only in lead position; prose provenance phrases (`based on`, `credit`, `adapted from`, `ported from`) do not justify at all. +- R3. A comment that carries a genuine non-obvious why and does not restate its adjacent code stays justified. `// why a clone: the SDK mutates the original in place` above `const buf = orig.slice();` passes today and must still pass. +- R4. The before-state and after-state are both re-derivable in-repo by one command, not by a transcript. +- R5. A `RestatesCode` conviction whose evidence cites nothing does not claim the comment restates the code. Measured: `// SAFETY: the SDK mutates this buffer in place, so a clone is required.` above `const buf = orig.slice();` is convicted with reason `restates what the code already says` and an empty citation, which is the reason string asserting a comparison the classifier never made. + +### Key Decisions + +- KD1. **Invert the precedence; do not delete the tables.** A why-comment is legitimate doctrine and the settled position is that the cap belongs on comment *content*, not on comment *length* or on the presence of a vocabulary. Deleting `NonObviousIntent` would cost the class the rule exists to protect (R3). Governs R1, R3. +- KD2. **Proven restatement is the only thing that outranks a marker.** Not the terminal empty-evidence rule. An unproven conviction must not be able to override a justification, or the change becomes "block everything" wearing a precedence patch. Governs R1. +- KD3. **The corpus is the specification.** Labels are re-derived and the per-kind floors re-run before the change is trusted; the F1 number after the change measures the new doctrine only if the labels moved with it. Governs R4. + +### Acceptance Examples + +- AE1. Marker on a proven restatement + - **Covers:** R1 + - **Given:** `// set x to 1 because` above `const x = 1;` + - **When:** the hook runs + - **Then:** exit 2, and the reason cites the shared tokens +- AE2. Marker on a non-restatement + - **Covers:** R3 + - **Given:** `// why a clone: the SDK mutates the original in place` above `const buf = orig.slice();` + - **When:** the hook runs + - **Then:** exit 0 — this passes today and must still pass after U2 +- AE4. Uncited conviction + - **Covers:** R5 + - **Given:** `// SAFETY: the SDK mutates this buffer in place, so a clone is required.` above `const buf = orig.slice();` + - **When:** the hook runs + - **Then:** exit 2 today, with reason `restates what the code already says` and no cited tokens. Target: not convicted on that reason. +- AE3. Prose provenance + - **Covers:** R2 + - **Given:** `// based on old code, sets x to 1` above `const x = 1;` + - **When:** the hook runs + - **Then:** exit 2 + +### Scope Boundaries + +**In scope** + +- `classify` precedence between `JUSTIFIED` and proven restatement +- `ATTRIBUTION_MARKERS` / `ATTRIBUTION_PREFIXES` narrowing +- corpus labels and per-kind floors for the affected kinds +- a re-runnable laundering probe fixture + +**Deferred** + +- Unsupported-extension coverage (a comment in `a.foo` is never seen) +- `INTENT_MARKERS` membership itself — precedence is the fix; trimming the list is a separate question the corpus should answer + +**Outside identity** + +- The npm launcher, release workflows, detector/tree-sitter layer + +--- + +## Planning Contract + +### Key Technical Decisions + +- KTD1. **Compute proven restatement before consulting `JUSTIFIED`, and let only a cited-evidence verdict pre-empt it.** `RestateEvidence::is_empty()` already distinguishes a cited claim from the terminal text-only path, so the pre-emption predicate exists and needs no new concept. `NarratesControlFlow` cites both verb and construct and pre-empts on the same footing. +- KTD2. **Keep the unreliable-context refusal intact.** On an `Edit`/`MultiEdit` fragment edge there is no adjacent code to prove anything against, so no pre-emption can fire there and the conservative fallback stands. This is why the fragment-edge fix lands first. +- KTD3. **Probe fixture is a test, not a script.** The 18-case laundering set becomes a table-driven test asserting a spare-count of 0 for the restatement rows, so the number in this plan cannot rot silently. + +### Assumptions + +- The five currently-blocked rows block for the restatement reason, not incidentally. Verified per-row by reading the cited reason, not the exit code alone. +- Corpus cases labelled `NonObviousIntent` mostly do not overlap their adjacent code; if many do, U4 grows and KD1 gets re-examined rather than forced. + +### Sequencing + +U0 has landed. U1 before U2 — the before-state must be pinned before the precedence moves. U3 after U2 so one behaviour change is measured at a time. U4 gates the claim, not the code. U5 last: it changes a reason string, and re-labelling in U4 must not be done against a label U5 is about to rename. + +--- + +## Implementation Units + +### U0. Fragment-edge unreliability (landed) + +- **Goal:** `Edit`/`MultiEdit` stops acquitting every added comment. +- **Requirements:** prerequisite for R1 on the edit path +- **Files:** `crates/comment-checker/src/check.rs`, `crates/comment-checker/tests/pipeline.rs` +- **Approach:** Mark context unreliable only where the fragment boundary could have removed the adjacent code: no adjacent code, or nothing after the comment. Restores the edge-scoped rule the adjudication plan specifies. +- **Verification:** `cargo test --all-targets` — 15 pipeline tests, F1 unchanged. Done. + +### U1. Laundering probe fixture + +- **Goal:** The before-state is a test, not a claim. +- **Requirements:** R4 +- **Dependencies:** U0 +- **Files:** + - `crates/comment-checker/tests/laundering.rs` — create +- **Approach:** Table of (comment, adjacent code, expected outcome) over the 18 rows, driven through `check` at the same seam `pipeline.rs` uses. Land it asserting the **current** outcomes, so U2's diff is the behaviour change and nothing else. +- **Verification:** `cargo test --test laundering` green before U2, and the row expectations flip in U2's diff. + +### U2. Proven restatement outranks a marker + +- **Goal:** A marker cannot acquit a comment whose restatement is proven. +- **Requirements:** R1, R3 +- **Dependencies:** U1 +- **Files:** + - `crates/comment-checker/src/classify.rs` — `classify` precedence +- **Approach:** Compute the context-aware verdict first when the context is reliable. If it is `NarratesControlFlow`, or `RestatesCode` whose evidence is non-empty, return it. Otherwise fall through to the existing `JUSTIFIED` → `UNNECESSARY` → terminal order unchanged. The unreliable path is untouched (KTD2). +- **Test scenarios:** + - AE1 blocks and the report cites shared tokens. + - AE2 passes. + - A linter directive on a restating line still passes — directives are machine-read and must not be convicted. + - A shebang and an SPDX header still pass. +- **Verification:** `cargo test --all-targets`, then `cargo mutants --file crates/comment-checker/src/classify.rs --timeout 90` at 100%. + +### U3. Narrow attribution to lead-position tags + +- **Goal:** Provenance prose stops acquitting. +- **Requirements:** R2 +- **Dependencies:** U2 +- **Files:** + - `crates/comment-checker/src/classify.rs` — `ATTRIBUTION_MARKERS`, `is_attribution` +- **Approach:** Keep `@author`, `@copyright`, `@see`, `@link` and require lead position, reusing the existing lead-strip helper. Drop `adapted from`, `based on`, `ported from`, `credit`. `@copyright` file headers remain justified through the license rule. +- **Test scenarios:** AE3 blocks; `// @see other-module` in lead position passes; `// see the other module` blocks. +- **Verification:** as U2, plus the F1 gate. + +### U4. Re-derive corpus labels and floors + +- **Goal:** The F1 number measures the new doctrine. +- **Requirements:** R4, KD3 +- **Dependencies:** U2, U3 +- **Files:** + - `eval/corpus.json` — re-adjudicate cases whose kind is `NonObviousIntent` or `Attribution` + - `crates/comment-checker/tests/f1.rs` — floors if a kind's bucket moved +- **Approach:** For each affected case, decide the label under the stated doctrine — does the comment restate its authored adjacent code? Record the count moved. Re-run the per-kind precision and recall floors; a kind that drops below its floor is a finding about the doctrine, not a number to lower. +- **Verification:** `cargo test --test f1` green with floors unchanged, or a written argument for any floor that moves. + +### U5. Stop claiming restatement without a citation + +- **Goal:** A conviction's reason names what the classifier actually found. +- **Requirements:** R5 +- **Dependencies:** U4 +- **Files:** + - `crates/comment-checker/src/comment.rs` — a terminal kind distinct from `RestatesCode` + - `crates/comment-checker/src/classify.rs` — terminal path returns it + - `crates/comment-checker/src/report.rs` — its reason string +- **Approach:** The terminal text-only path keeps convicting — default-deny is the forcing behaviour and is not in question. It stops borrowing the `RestatesCode` label it cannot evidence, and reports the reason it can stand behind: the comment carries prose with no counterpart in the adjacent code. Verdict unchanged, claim honest. This is the same discipline U2 applies to acquittal, applied to the reason string. +- **Test scenarios:** AE4 blocks with the new reason and no `shares` clause; `report_cites_restate_evidence` still shows `shares counter` and `increment ↔ +=` for a real overlap. +- **Verification:** as U2, plus check 4. + +--- + +## Verification Contract + +| # | Check | Applies | Done signal | +|---|---|---|---| +| 1 | `cargo fmt --check && cargo clippy --all-targets -- -D warnings && cargo test --all-targets` | all | one-shot repo gate green | +| 2 | `cargo test --test laundering` | U1, U2, U3 | zero spares on marker-plus-restatement rows | +| 3 | `cargo mutants --file crates/comment-checker/src/classify.rs --timeout 90` | U2, U3 | 100%, per the project rule for classifier changes | +| 4 | `cargo test --test f1` | U4 | per-kind floors hold on re-derived labels | +| 5 | `cargo test --test laundering -- ae4` | U5 | AE4 reason carries no `shares` clause | + +--- + +## Definition of Done + +- [ ] U1: probe fixture in-repo; the 18 rows are a test. +- [ ] U2: proven restatement pre-empts `JUSTIFIED`; AE1 blocks, AE2 passes. +- [ ] U3: attribution is lead-position tags only; AE3 blocks. +- [ ] U4: corpus re-adjudicated; floors hold or the move is argued. +- [ ] U5: no conviction claims restatement without citing it. +- [ ] Repo one-shot gate and classifier mutation both green. + +## Risks + +- **Over-conviction of legitimate why-comments.** The pre-emption is gated on *cited* evidence (KTD2), so a comment with no token overlap cannot be pre-empted. AE2 is the guard, and U1 pins it before the change. +- **Corpus churn masquerading as a win.** U4 can produce a better F1 by relabelling toward the new rule. Mitigation: the per-kind floors are re-run unchanged, and a floor that has to move is reported rather than edited. +- **Mutation score regression.** Reordering `classify` adds a branch that mutants will probe. Check 3 is the gate; a surviving mutant means a missing test, not a threshold to relax. +- **Precedence inversion changes directive handling.** A linter directive that restates its line must stay justified. U2's third test scenario is the guard; if it fails, the pre-emption needs a machine-read exemption ahead of it. From 8bb75eb1dce7b14fda1cc193f649a5783c62ade7 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Mon, 24 Aug 2026 22:18:47 +0000 Subject: [PATCH 3/5] test(edit-path): gate the corpus through the Edit seam MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit f1.rs drives the Write path only, so the Edit/MultiEdit path carried no corpus coverage — which is how a blanket fragment acquittal shipped with every gate green. Two invariants over the 60 labelled cases, driven through check() as Edit payloads: no justified case blocks, no unnecessary case is spared. Verified to fail on the pre-fix check.rs with 17 spared cases (15 RestatesCode, 2 NarratesControlFlow) and pass with the fix. --- crates/comment-checker/tests/edit_path.rs | 60 +++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 crates/comment-checker/tests/edit_path.rs diff --git a/crates/comment-checker/tests/edit_path.rs b/crates/comment-checker/tests/edit_path.rs new file mode 100644 index 0000000..48697e3 --- /dev/null +++ b/crates/comment-checker/tests/edit_path.rs @@ -0,0 +1,60 @@ +//! The Edit-path gate: every corpus case driven through `check` as an `Edit` +//! payload. `f1.rs` exercises the `Write` path only, so before this gate the +//! `Edit`/`MultiEdit` path carried no corpus coverage — which is how a blanket +//! fragment acquittal shipped while every other gate stayed green. + +mod common; + +use claude_code_comment_checker::{Outcome, check}; +use common::{Case, Label, load_corpus, synthesize_source, synthesized_path}; + +fn escape(s: &str) -> String { + s.replace('\\', "\\\\") + .replace('"', "\\\"") + .replace('\n', "\\n") +} + +/// The realistic edit: the file held the case's code, and the agent's edit adds +/// the comment above, below, or beside it. +fn edit_payload(case: &Case) -> String { + let file_path = synthesized_path(case); + let old = escape(&format!("{}\n", case.code)); + let new = escape(&synthesize_source(case)); + format!( + r#"{{"tool_name":"Edit","tool_input":{{"file_path":"{file_path}","old_string":"{old}","new_string":"{new}"}}}}"# + ) +} + +fn blocks(case: &Case) -> bool { + matches!(check(&edit_payload(case), ""), Outcome::Block { .. }) +} + +#[test] +fn edit_path_never_blocks_a_justified_comment() { + let offenders: Vec<_> = load_corpus() + .into_iter() + .filter(|case| case.label == Label::Justified && blocks(case)) + .map(|case| format!("[{}/{}] {}", case.kind, case.language, case.text)) + .collect(); + assert!( + offenders.is_empty(), + "{} justified corpus case(s) blocked on the Edit path:\n {}", + offenders.len(), + offenders.join("\n ") + ); +} + +#[test] +fn edit_path_catches_every_unnecessary_corpus_case() { + let missed: Vec<_> = load_corpus() + .into_iter() + .filter(|case| case.label == Label::Unnecessary && !blocks(case)) + .map(|case| format!("[{}/{}] {}", case.kind, case.language, case.text)) + .collect(); + assert!( + missed.is_empty(), + "{} unnecessary corpus case(s) spared on the Edit path:\n {}", + missed.len(), + missed.join("\n ") + ); +} From 828a8d0fa695a650194f75b47dfc1466056c2ff7 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Mon, 24 Aug 2026 22:23:22 +0000 Subject: [PATCH 4/5] test(common): one payload builder for both check-seam suites pipeline.rs inlined a JSON escaper and edit_path.rs added a byte-identical copy. Both now call common::write_payload / common::edit_payload, so an escaping bug is fixed once. Applied from the reuse review. --- crates/comment-checker/tests/common/mod.rs | 22 ++++++++++++++++++ crates/comment-checker/tests/edit_path.rs | 26 ++++++---------------- crates/comment-checker/tests/pipeline.rs | 12 +++------- 3 files changed, 32 insertions(+), 28 deletions(-) diff --git a/crates/comment-checker/tests/common/mod.rs b/crates/comment-checker/tests/common/mod.rs index 4656682..3b74a4e 100644 --- a/crates/comment-checker/tests/common/mod.rs +++ b/crates/comment-checker/tests/common/mod.rs @@ -16,6 +16,28 @@ use claude_code_comment_checker::{ }; use serde::Deserialize; +fn escape(field: &str) -> String { + field + .replace('\\', "\\\\") + .replace('"', "\\\"") + .replace('\n', "\\n") +} + +pub fn write_payload(file_path: &str, content: &str) -> String { + let content = escape(content); + format!( + r#"{{"tool_name":"Write","tool_input":{{"file_path":"{file_path}","content":"{content}"}}}}"# + ) +} + +pub fn edit_payload(file_path: &str, old_string: &str, new_string: &str) -> String { + let old_string = escape(old_string); + let new_string = escape(new_string); + format!( + r#"{{"tool_name":"Edit","tool_input":{{"file_path":"{file_path}","old_string":"{old_string}","new_string":"{new_string}"}}}}"# + ) +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum Label { Unnecessary, diff --git a/crates/comment-checker/tests/edit_path.rs b/crates/comment-checker/tests/edit_path.rs index 48697e3..093a75d 100644 --- a/crates/comment-checker/tests/edit_path.rs +++ b/crates/comment-checker/tests/edit_path.rs @@ -6,27 +6,15 @@ mod common; use claude_code_comment_checker::{Outcome, check}; -use common::{Case, Label, load_corpus, synthesize_source, synthesized_path}; - -fn escape(s: &str) -> String { - s.replace('\\', "\\\\") - .replace('"', "\\\"") - .replace('\n', "\\n") -} - -/// The realistic edit: the file held the case's code, and the agent's edit adds -/// the comment above, below, or beside it. -fn edit_payload(case: &Case) -> String { - let file_path = synthesized_path(case); - let old = escape(&format!("{}\n", case.code)); - let new = escape(&synthesize_source(case)); - format!( - r#"{{"tool_name":"Edit","tool_input":{{"file_path":"{file_path}","old_string":"{old}","new_string":"{new}"}}}}"# - ) -} +use common::{Case, Label, edit_payload, load_corpus, synthesize_source, synthesized_path}; fn blocks(case: &Case) -> bool { - matches!(check(&edit_payload(case), ""), Outcome::Block { .. }) + let payload = edit_payload( + &synthesized_path(case), + &format!("{}\n", case.code), + &synthesize_source(case), + ); + matches!(check(&payload, ""), Outcome::Block { .. }) } #[test] diff --git a/crates/comment-checker/tests/pipeline.rs b/crates/comment-checker/tests/pipeline.rs index eff33d8..c4541ff 100644 --- a/crates/comment-checker/tests/pipeline.rs +++ b/crates/comment-checker/tests/pipeline.rs @@ -3,15 +3,9 @@ use claude_code_comment_checker::{Outcome, check}; -fn write(file_path: &str, content: &str) -> String { - let content = content - .replace('\\', "\\\\") - .replace('"', "\\\"") - .replace('\n', "\\n"); - format!( - r#"{{"tool_name":"Write","tool_input":{{"file_path":"{file_path}","content":"{content}"}}}}"# - ) -} +mod common; + +use common::write_payload as write; #[test] fn unnecessary_comment_blocks() { From 6fd346a471c280c143068d9aa5f726ec5c0dafe6 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Mon, 24 Aug 2026 22:26:21 +0000 Subject: [PATCH 5/5] docs(solutions): why three green gates missed an inert edit path Coverage composes over stage x entry path: the mutation gate pinned the pure classifier while the defect sat in the adapter upstream of it, the labelled corpus only ever constructed the whole-content payload, and two composition tests had pinned the blanket acquittal as the contract. --- ...gment-trust-flag-scope-exceeds-the-edge.md | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 docs/solutions/logic-errors/fragment-trust-flag-scope-exceeds-the-edge.md diff --git a/docs/solutions/logic-errors/fragment-trust-flag-scope-exceeds-the-edge.md b/docs/solutions/logic-errors/fragment-trust-flag-scope-exceeds-the-edge.md new file mode 100644 index 0000000..4e50873 --- /dev/null +++ b/docs/solutions/logic-errors/fragment-trust-flag-scope-exceeds-the-edge.md @@ -0,0 +1,103 @@ +--- +title: A trust flag set for a whole fragment instead of its edge disables every context-aware rule on that path +date: 2026-08-24 +category: logic-errors +module: hook payload adaptation (Edit/MultiEdit fragment context) and the classifier's conservative fallback +problem_type: logic_error +component: tooling +severity: high +symptoms: + - "An added comment that restates its adjacent code blocks on a Write payload and passes on an Edit payload carrying byte-identical text" + - "Only the text-only rules ever fire on Edit/MultiEdit: AgentMemo, CommentedOutCode, VacuousTodo. RestatesCode and NarratesControlFlow never appear" + - "No test failed, no gate reddened, mutation score on the classifier stayed at 100%, and F1 stayed above its floor" + - "The hook exits 0 on the tool an agent actually uses to modify an existing file, so the gate is inert exactly where it is load-bearing" +root_cause: scope_issue +resolution_type: code_fix +related_components: + - testing_framework + - development_workflow +tags: [gate-coverage, entry-path, trust-flag, fragment-edge, mutation-blind-spot, conservative-fallback, functional-core-shell] +--- + +# A conservative fallback is an acquittal channel; its guard's scope is the enforcement boundary + +## Problem + +`check` adapts three payload shapes into one comment stream. For `Write` it detects comments in the whole content; for `Edit` and `MultiEdit` it diffs comment sets and keeps only the newly-added ones, then stamps each surviving comment's `CommentContext` as untrusted. `classify` reads that flag twice: `reliable_adjacent` refuses to hand adjacent code to the context-aware detectors, and the terminal fallback rewrites an evidence-free `RestatesCode` verdict into `Justification::NonObviousIntent`. + +Stamping the flag for every comment in the fragment therefore did not make the edit path *conservative*. It made the edit path *unenforced* for every rule that needs adjacent code — which is every rule the tool exists for. + +## Mechanism + +Let $R$ be the rule set, partitioned into text-only rules $R_t$ (decidable from the comment string) and context rules $R_c$ (requiring adjacent code). Let $u(c)$ be the trust flag on comment $c$, and $p$ the entry path. + +The classifier's fallback gives, for any $c$: + +$$ +u(c) = \text{true} \;\Rightarrow\; \text{verdict}(c) \in R_t \cup \{\text{Justified}\} +$$ + +The adapter set $u(c) = \text{true}$ unconditionally for all $c$ on the edit paths. Substituting: + +$$ +\forall c \in p_{\text{edit}}:\; u(c) = \text{true} \;\Rightarrow\; R_{\text{effective}}(p_{\text{edit}}) = R_t +$$ + +$R_c$ is dead code on that path — not weakened, absent. A conservative default composed with an unconditional guard is not caution; it is a blanket acquittal wearing caution's name. + +The soundness condition the flag was *meant* to encode is narrower. A fragment is a contiguous slice of file text. If $c$ has a following code sibling inside the slice, that sibling is also $c$'s following sibling in the whole file — the boundary cannot have inserted code between them. So trust is warranted whenever the side $c$ annotates is present within the slice, and unwarranted only in two shapes: no adjacent code captured at all, or the detector could read $c$ only as a trailing comment, which `derive_context` yields exactly when the next code sibling is absent. Hence: + +$$ +u(c) \;=\; \neg\,\text{adjacent}(c) \;\lor\; \text{position}(c) = \text{Trailing} +$$ + +## Why every gate stayed green + +Three independent gates covered this subsystem and none could see the defect. + +1. **Mutation coverage measured the wrong stage.** The mutation gate targets the pure classifier. The defect lived in the adapter that constructs the classifier's inputs. A pure core can be perfectly pinned while the shell feeding it supplies uniformly degraded inputs; every mutant of the core still dies, because the tests that kill them construct their contexts directly. + +2. **The labelled corpus drove one entry path.** The evaluation harness synthesises a source snippet per case and runs it as a whole-content parse — the `Write` shape. Coverage of behaviour on the paths the harness never constructs is zero regardless of corpus size, label quality, or per-kind floors. Formally, with $P$ the entry paths and $G \subseteq P$ those the gate drives, a defect confined to $P \setminus G$ is unobservable through that gate. + +3. **The composition tests asserted the defect as the contract.** Two tests named the blanket acquittal as intended behaviour and passed. A test that encodes the bug is worse than no test: it converts a defect into a protected invariant and makes the fix look like a regression. + +The union of the three is the general trap: **a gate suite can be individually sound at every layer and still leave an entry path with no coverage at all**, because coverage composes over (stage $\times$ path), not over stages alone. + +## Architectural Invariants + +- **Guard scope equals the condition it names.** A flag called "the boundary may have removed context" must be true only where the boundary could have removed context. When a guard's scope exceeds its predicate, the excess is silent capability loss, not conservatism. Corollary: an unconditional assignment to a trust flag is always a bug or a constant. + +- **Every acquittal channel is an enforcement hole with a scope.** Enumerate the ways a verdict can become Justified — matched exemption, fallback rewrite, unsupported input — and state the scope of each. A fallback that manufactures a specific justification for an unprovable case is indistinguishable, downstream, from having proven it. + +- **Coverage composes over stage × entry path.** Mutation-testing a pure core certifies the core, not the adapters upstream of it. A labelled corpus certifies the paths it constructs, not the paths it does not. Any input shape the production entrypoint accepts is an axis the gate must enumerate. + +- **A conservative default must degrade to a floor, not to the empty set.** State the floor explicitly and assert it. "Falls back to the text-only rules" is a floor; "falls back to Justified for everything unmatched" is an off switch. + +- **A test that passes on the pre-fix artifact certifies nothing.** Gate authorship is only complete once the gate has been run against the defective revision and observed to fail. A gate never shown red is an untested assertion about an untested property. + +## Verification + +The falsifying observation to demand: identical comment text on two payload shapes must reach identical verdicts wherever the fragment supplies the annotated side. + +| Probe | Expectation | +| --- | --- | +| restatement with code following it, inside the fragment | blocks, reason cites the shared tokens | +| same text as whole-content payload | blocks identically — the two paths must not disagree | +| comment at the fragment tail, nothing after it | passes — the genuine truncated edge | +| comment with no adjacent code in the fragment | passes — neither side captured | +| machine-read directive on a restating line | passes on both paths | + +Session verification: the labelled corpus driven through the edit seam moved from 43 of 60 correct to 60 of 60. Seventeen unnecessary cases had been spared — fifteen restatement, two flow-narration — and zero justified cases were newly blocked, so the change carries no false-positive cost against the corpus. The whole-content path was unchanged on all sixty, confirming the fix is confined to the adapted paths. Operator mutation of the new predicate — disjunction to conjunction, equality to inequality, and each constant — is killed by the composition tests and the new edit-path gate. + +## Prevention + +- Enumerate the entry shapes the production entrypoint accepts, then drive the labelled corpus through each. Gate: per-path corpus invariants asserting no justified case blocks and no unnecessary case is spared. +- Before trusting a new gate, run it against the revision that contained the defect and require a red result naming the affected cases. Gate: the gate's own commit message records the pre-fix failure count. +- When a fallback rewrites a verdict, assert the floor it degrades to, not merely that it does not crash. Gate: a test per path asserting a context rule still fires where context is present. +- Treat an unconditional write to a trust or capability flag as a review stop. The predicate belongs in the assignment, not in the reader. +- When a test's name asserts that a detector is *not* consulted, verify that this is a decision and not a discovered behaviour that was pinned. Gate: this document. + +## Related + +- docs/solutions/design-patterns/evidence-gated-context-aware-classification.md — the evidence discipline applied to convictions; this document is the same discipline applied to acquittals, which that design left on an unconditional guard. +- docs/solutions/integration-issues/cached-gate-task-scope-exceeds-determinants.md — the sibling scope-mismatch failure. There a hash surface exceeded its determinants and destroyed cache value; here a guard's scope exceeded its predicate and destroyed enforcement. Both are audited by comparing a declared scope against the set it claims to track, in both directions.