diff --git a/CHANGELOG.md b/CHANGELOG.md index c84beba..d0e2c00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,63 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +Accuracy fixes from the 2026-08-10 library audit: three numeric-claim +false-positive classes stopped, two silent false-negative classes closed +(short count labels, single-span/fenced CLI flags), markdown link titles +handled, and the semantic layer now judges against real source passages +instead of the content itself. Every fix carries a regression corpus case. + +### Fixed + +- **Comma-grouped numbers are one claim.** "1,234 tests" previously + extracted the "234" fragment and flagged an error-severity count + mismatch against `tests=1234`; `1,234` is now extracted as the single + value 1234 (commas stripped). Grouped values also stay checked when + year-valued — "2,026 widgets" is a count, never a year. +- **Decimals and version components are not counts.** "94.53" near a + source keyword extracted 94 and 53 as separate claims and flagged both; + the "10" in "Python 3.10" was flagged against a nearby source. Digit + runs adjacent to a decimal point are now skipped. +- **Short count-source labels are no longer silently dead.** Label words + of length ≤ 3 ("api", "eps") were dropped by the keyword filter, so + those sources could never match any claim — a silent false negative. + Short words now require an exact word-boundary match on both sides + ("api" matches "api" but not "rapid"), and participate only when the + label has no longer word — in a mixed label ("number of tests") a short + word is usually a stopword, and letting "of" match ordinary prose would + drag unrelated numbers into the source. Longer words keep the + leading-boundary match so plural drift still hits. +- **Flags inside real command spans and shell fences are now checked.** + The extractor only matched a flag backticked alone (`` `--flag` ``); + the common form `` `mytool --flag` `` — the whole command in one span — + matched nothing, despite a comment claiming it was handled, and flags + inside ```` ```bash ```` fences were unchecked. Both are now extracted + (fences: bash/sh/shell/console/zsh), with the command guessed from the + span or line itself before falling back to preceding prose. +- **Markdown link titles no longer break path checks.** + `[text](docs/a.md "Read me")` treated the whole `docs/a.md "Read me"` + string as the path and errored even when the file exists. The optional + title is stripped and `` targets are unwrapped; dead + paths with titles are still flagged. +- **The semantic layer no longer judges content against itself.** + `verify()` passed the generated content as its own source passages — + vacuously faithful by construction. `VerifyContext` gains a `passages` + field that is forwarded to the judge; when `semantic=True` with no + passages, verify degrades gracefully (warning, judge not called). +- **The no-judge warning no longer misreports a bad judge.** A judge that + was provided but fails the `Judge` protocol check was reported as "no + judge was provided"; the message now names the failing object and the + protocol mismatch. + +### Changed + +- Import resolution passes the module name to the child interpreter via + `argv` instead of f-string interpolation into the `-c` program — + defense in depth (`ast` already guarantees identifier-safe names). +- Corpus flag cases use the realistic single-span form + (`` `mytool --verbose` ``) instead of the unnatural split form; the + split form remains covered by `evasion_substring_flag`. + ## [0.2.2] - 2026-07-17 Accuracy + reliability patch from a full library review: five silent diff --git a/src/attune_verify/_extract.py b/src/attune_verify/_extract.py index 568c6a1..ffe0c9e 100644 --- a/src/attune_verify/_extract.py +++ b/src/attune_verify/_extract.py @@ -42,7 +42,15 @@ class NumericClaim: re.MULTILINE | re.DOTALL, ) _LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)]+)\)") -_NUM_RE = re.compile(r"\b(\d{2,})\b") # 2+ digit numbers (skip single digits) +# 2+ digit numbers (skip single digits). Comma-grouped values ("1,234") are one +# claim — the first alternative captures the whole group before the bare \d{2,} +# can grab a fragment. Digit runs touching a decimal point ("94.53", the "10" +# in "Python 3.10") are decimal/version components, not counts, and are skipped +# via the surrounding lookarounds. +_NUM_RE = re.compile(r"(?. +_LINK_TITLE_RE = re.compile(r"""^(\S+)\s+("[^"]*"|'[^']*'|\([^)]*\))$""") def extract_code_fences(content: str) -> List[CodeFence]: @@ -63,22 +71,42 @@ def extract_code_fences(content: str) -> List[CodeFence]: def extract_links(content: str) -> List[MarkdownLink]: - """Extract all markdown links from content.""" + """Extract all markdown links from content. + + Targets are normalized: an optional markdown title + (``docs/a.md "Read me"``) is stripped and ```` wrapping is + removed, so checkers see only the path. + """ links = [] for match in _LINK_RE.finditer(content): line = content[: match.start()].count("\n") + 1 links.append( MarkdownLink( text=match.group(1), - target=match.group(2), + target=_clean_link_target(match.group(2)), line=line, ) ) return links +def _clean_link_target(raw: str) -> str: + """Strip an optional title and angle-bracket wrapping from a link target.""" + target = raw.strip() + title_match = _LINK_TITLE_RE.match(target) + if title_match: + target = title_match.group(1) + if target.startswith("<") and target.endswith(">"): + target = target[1:-1] + return target + + def extract_numeric_claims(content: str) -> List[NumericClaim]: - """Extract numeric claims (2+ digit numbers) with surrounding context.""" + """Extract numeric claims (2+ digit numbers) with surrounding context. + + Comma-grouped numbers ("1,234") are one claim with the commas stripped; + decimal and version components ("94.53", "Python 3.10") are not claims. + """ claims = [] for match in _NUM_RE.finditer(content): line = content[: match.start()].count("\n") + 1 @@ -86,7 +114,7 @@ def extract_numeric_claims(content: str) -> List[NumericClaim]: end = min(len(content), match.end() + 40) claims.append( NumericClaim( - value=int(match.group(1)), + value=int(match.group(1).replace(",", "")), context=content[start:end].replace("\n", " "), line=line, ) diff --git a/src/attune_verify/_verify.py b/src/attune_verify/_verify.py index 980e74e..448f04f 100644 --- a/src/attune_verify/_verify.py +++ b/src/attune_verify/_verify.py @@ -25,7 +25,8 @@ def verify(content: str, context: VerifyContext) -> VerifyResult: Deterministic checkers always run and are independent — a failure in one does not abort the others. The semantic layer runs only when - context.semantic is True and a judge is available. + context.semantic is True and both a judge and source passages are + available. Args: content: LLM-generated content to verify. @@ -108,14 +109,31 @@ def _run_semantic(result: VerifyResult, content: str, context: VerifyContext) -> """Run the semantic layer if a judge is available.""" from attune_verify.semantic.protocol import Judge # noqa: PLC0415 - if context.judge is None or not isinstance(context.judge, Judge): + if context.judge is None: + detail = ( + "Semantic layer requested (context.semantic=True) " + "but no judge was provided in VerifyContext.judge" + ) + elif not isinstance(context.judge, Judge): + detail = ( + "Semantic layer requested (context.semantic=True) but the " + f"provided judge ({type(context.judge).__name__}) does not " + "satisfy the Judge protocol (missing a compatible score())" + ) + elif not context.passages: + # Without independent source passages the judge would score the + # content against itself — vacuously faithful, so skip instead. + detail = ( + "Semantic layer requested (context.semantic=True) but no " + "source passages were provided in VerifyContext.passages" + ) + else: + detail = None + if detail is not None: result.findings.append( Finding( kind=FindingKind.SEMANTIC, - detail=( - "Semantic layer requested (context.semantic=True) " - "but no judge was provided in VerifyContext.judge" - ), + detail=detail, evidence="", severity="warning", ) @@ -126,7 +144,7 @@ def _run_semantic(result: VerifyResult, content: str, context: VerifyContext) -> verdict = context.judge.score( query="Verify this generated content for faithfulness", answer=content, - passages=content, + passages=context.passages, ) result.semantic_ran = True if not verdict.faithful: diff --git a/src/attune_verify/checkers/counts.py b/src/attune_verify/checkers/counts.py index b053aa7..2d83fdc 100644 --- a/src/attune_verify/checkers/counts.py +++ b/src/attune_verify/checkers/counts.py @@ -83,12 +83,38 @@ def _find_close_label( """ context_lower = context.lower() for label in sources: - words = label.lower().split() - if any(re.search(rf"\b{re.escape(w)}", context_lower) for w in words if len(w) > 3): + if any(_word_matches(w, context_lower) for w in _match_words(label)): return label return None +def _match_words(label: str) -> list[str]: + """Return the label words eligible for keyword matching. + + Short words (<= 3 chars) participate only when the label has no longer + word to match on. A label like "api" was silently dead when short words + were dropped outright; but in a mixed label like "number of tests" a + short word is usually a stopword — letting "of" match ordinary prose + would drag unrelated numbers into the source. + """ + words = label.lower().split() + long_words = [w for w in words if len(w) > 3] + return long_words if long_words else words + + +def _word_matches(word: str, text: str) -> bool: + """Match a label keyword in text, calibrated to the keyword's length. + + Long words match on a leading boundary only, so plural drift still hits + ("test" ~ "tests"). Short words ("api", "eps") prefix-match far too + loosely, so they require an exact word-boundary match on both sides — + previously they were dropped entirely, silently killing their source. + """ + if len(word) > 3: + return re.search(rf"\b{re.escape(word)}", text) is not None + return re.search(rf"\b{re.escape(word)}\b", text) is not None + + def _year_like(value: int) -> bool: return _YEAR_MIN <= value <= _YEAR_MAX @@ -99,10 +125,11 @@ def _label_follows_number(claim: NumericClaim, label: str) -> bool: "2026 widgets" reads as a widget count; "2026 versions of the widgets" reads as a year that merely has the keyword nearby. """ - match = re.search(rf"\b{claim.value}\b((?:\s+\S+){{1,2}})", claim.context.lower()) + # The claim value has commas stripped, but the context still shows the + # written form — match either ("2,026 widgets" is a count, never a year). + grouped = re.escape(f"{claim.value:,}") + match = re.search(rf"\b(?:{claim.value}|{grouped})\b((?:\s+\S+){{1,2}})", claim.context.lower()) if match is None: return False following = match.group(1) - return any( - re.search(rf"\b{re.escape(w)}", following) for w in label.lower().split() if len(w) > 3 - ) + return any(_word_matches(w, following) for w in _match_words(label)) diff --git a/src/attune_verify/checkers/flags.py b/src/attune_verify/checkers/flags.py index 3651c79..ee9bfc4 100644 --- a/src/attune_verify/checkers/flags.py +++ b/src/attune_verify/checkers/flags.py @@ -4,11 +4,18 @@ import re import subprocess -from typing import Dict, FrozenSet, List +from typing import Dict, FrozenSet, List, Optional +from attune_verify._extract import _FENCE_RE, extract_code_fences from attune_verify.result import Finding, FindingKind -_FLAG_RE = re.compile(r"`(--[\w-]+)`") +# One inline code span (`mytool --flag`); fences are handled separately. +_INLINE_CODE_RE = re.compile(r"`([^`\n]+)`") +# A flag token anywhere in code text. Stops before "=value"; the negative +# lookbehind keeps it from matching the tail of a longer flag or a "---" rule. +_FLAG_TOKEN_RE = re.compile(r"(? Optional[Finding]: + """Check one flag against its command's help; None when it verifies.""" + help_text = _get_help(cmd, help_commands, allowed_help_cmds) + if help_text is None: + return Finding( + kind=FindingKind.UNKNOWN_FLAG, + detail=( + f"Flag '{flag}' could not be verified " + f"(no --help output available for command '{cmd}')" + ), + evidence=evidence, + severity="warning", + ) + if not _flag_in_help(flag, help_text): + return Finding( + kind=FindingKind.UNKNOWN_FLAG, + detail=f"Flag '{flag}' not found in '{cmd} --help'", + evidence=evidence, + severity="error", + ) + return None + + def _flag_in_help(flag: str, help_text: str) -> bool: """Return True if flag appears in help as a whole token. diff --git a/src/attune_verify/checkers/imports.py b/src/attune_verify/checkers/imports.py index 7b6863b..88fc4b8 100644 --- a/src/attune_verify/checkers/imports.py +++ b/src/attune_verify/checkers/imports.py @@ -105,12 +105,19 @@ def _modules_from_node(node: ast.AST) -> list[str]: def _resolves(module: str, env_python: str) -> bool: - """Return True if module is importable in env_python.""" + """Return True if module is importable in env_python. + + The module name travels via argv, not f-string interpolation into the + -c program — defense in depth (ast already guarantees identifier-safe + names, but the child program must not depend on that). + """ result = subprocess.run( [ env_python, "-c", - f"import importlib.util; " f"print(importlib.util.find_spec('{module}') is not None)", + "import importlib.util, sys; " + "print(importlib.util.find_spec(sys.argv[1]) is not None)", + module, ], capture_output=True, text=True, diff --git a/src/attune_verify/context.py b/src/attune_verify/context.py index 87856af..c437022 100644 --- a/src/attune_verify/context.py +++ b/src/attune_verify/context.py @@ -8,7 +8,7 @@ import sys from dataclasses import dataclass, field from pathlib import Path -from typing import TYPE_CHECKING, Callable, Dict, FrozenSet, Optional, Union +from typing import TYPE_CHECKING, Callable, Dict, FrozenSet, List, Optional, Union if TYPE_CHECKING: from attune_verify.semantic.protocol import Judge @@ -36,7 +36,11 @@ class VerifyContext: judge: Optional semantic judge implementing the Judge protocol. Required for the semantic layer; when absent and semantic=True, verify degrades gracefully (warning, not error). - semantic: Enable the LLM semantic layer. Requires a judge. + passages: Source-of-truth passage(s) the semantic judge grounds the + content against. Required for the semantic layer — judging content + against itself is vacuous, so when absent and semantic=True, + verify degrades gracefully (warning, not error). + semantic: Enable the LLM semantic layer. Requires a judge and passages. """ project_root: Optional[Path] = None @@ -45,4 +49,5 @@ class VerifyContext: allowed_help_cmds: FrozenSet[str] = field(default_factory=frozenset) count_sources: Dict[str, Union[int, Callable[[], int]]] = field(default_factory=dict) judge: Optional["Judge"] = None + passages: Optional[Union[str, List[str]]] = None semantic: bool = False diff --git a/tests/corpus/cases.py b/tests/corpus/cases.py index 7443053..14c542e 100644 --- a/tests/corpus/cases.py +++ b/tests/corpus/cases.py @@ -91,7 +91,14 @@ def _py(code: str) -> str: CorpusCase( name="clean_flag", label="clean", - content="Run `mytool` `--verbose` for detailed output.", + # Realistic form: the whole command lives in ONE backtick span. + content="Run `mytool --verbose` for detailed output.", + help_commands={"mytool": "Options:\n --verbose Be loud\n --help Show help\n"}, + ), + CorpusCase( + name="clean_flag_bash_fence", + label="clean", + content="Run it:\n```bash\n$ mytool --verbose\n```\n", help_commands={"mytool": "Options:\n --verbose Be loud\n --help Show help\n"}, ), # --------------------------------------------------------- hallucinated @@ -124,7 +131,10 @@ def _py(code: str) -> str: CorpusCase( name="fake_flag", label="hallucinated", - content="Pass `mytool` `--nonexistent` to enable it.", + # Realistic form: the whole command lives in ONE backtick span — the + # v0.2.2 extractor only matched a flag backticked alone, so this + # (the most common way LLMs write commands) passed silently. + content="Pass `mytool --nonexistent` to enable it.", help_commands={"mytool": "Options:\n --verbose Be loud\n --help Show help\n"}, expected=(ExpectedFinding(FindingKind.UNKNOWN_FLAG, "--nonexistent"),), ), @@ -195,4 +205,112 @@ def _py(code: str) -> str: count_sources={"tests": 50, "modules": 12}, expected=(ExpectedFinding(FindingKind.COUNT_MISMATCH, "12"),), ), + # ------------------------------------------------- 2026-08-10 audit fixes + CorpusCase( + name="clean_comma_grouped_count", + label="clean", + # "1,234" is ONE number. The v0.2.2 extractor grabbed the "234" + # fragment and flagged it against tests=1234 (error false positive). + content="The suite runs 1,234 tests on every push.", + count_sources={"tests": 1234}, + ), + CorpusCase( + name="comma_grouped_count_mismatch", + label="hallucinated", + # Comma handling must not cost recall: a wrong grouped count is + # still a mismatch. + content="The suite runs 1,234 tests on every push.", + count_sources={"tests": 999}, + expected=(ExpectedFinding(FindingKind.COUNT_MISMATCH, "1234"),), + ), + CorpusCase( + name="comma_grouped_year_valued_count", + label="hallucinated", + # "2,026" is year-VALUED but comma-grouped — nobody writes a year + # with a thousands separator, so it is a count and must be checked. + content="There are 2,026 widgets in stock.", + count_sources={"widgets": 12}, + expected=(ExpectedFinding(FindingKind.COUNT_MISMATCH, "2026"),), + ), + CorpusCase( + name="clean_decimal_near_keyword", + label="clean", + # "94.53" is a decimal, not two counts — the v0.2.2 extractor split + # it into 94 and 53 and flagged both against the source. + content="Coverage sits at 94.53 percent across the tests.", + count_sources={"tests": 50}, + ), + CorpusCase( + name="clean_version_number_near_keyword", + label="clean", + # The "10" in "Python 3.10" is a version component, not a module + # count — a nearby source keyword must not turn it into a claim. + content="Requires Python 3.10 to load the python modules.", + count_sources={"python modules": 12}, + ), + CorpusCase( + name="short_label_count_mismatch", + label="hallucinated", + # v0.2.2 dropped label words of length <= 3, so an "api" source + # could never match any claim — silently dead (false negative). + content="The service exposes 42 api endpoints.", + count_sources={"api": 3}, + expected=(ExpectedFinding(FindingKind.COUNT_MISMATCH, "42"),), + ), + CorpusCase( + name="clean_short_label_boundary", + label="clean", + # Short labels match on exact word boundaries only — "api" must not + # match inside "rapid" and drag unrelated numbers into the source. + content="We shipped 42 rapid iterations this quarter.", + count_sources={"api": 3}, + ), + CorpusCase( + name="clean_stopword_in_mixed_label", + label="clean", + # A short word in a MIXED label is usually a stopword: "of" must not + # match ordinary prose and drag unrelated numbers into the source. + # Short-word matching is a fallback for all-short labels only. + content="Only 12 of the widgets remain in the box.", + count_sources={"number of tests": 83}, + ), + CorpusCase( + name="mixed_label_long_word_still_matches", + label="hallucinated", + # The fallback rule must not cost recall: a mixed label still + # matches on its long words. + content="The suite ran 12 tests successfully.", + count_sources={"number of tests": 83}, + expected=(ExpectedFinding(FindingKind.COUNT_MISMATCH, "12"),), + ), + CorpusCase( + name="evasion_flag_in_bash_fence", + label="evasion", + # Flags inside ```bash fences were entirely unchecked in v0.2.2. + content="Enable it:\n```bash\nmytool --nonexistent\n```\n", + help_commands={"mytool": "Options:\n --verbose Be loud\n --help Show help\n"}, + expected=(ExpectedFinding(FindingKind.UNKNOWN_FLAG, "--nonexistent"),), + ), + CorpusCase( + name="clean_link_with_title", + label="clean", + # Markdown title syntax: the title is not part of the path — v0.2.2 + # checked 'docs/a.md "Read me"' for existence and errored. + content='See [the doc](docs/a.md "Read me") for details.', + files=("docs/a.md",), + ), + CorpusCase( + name="clean_link_angle_brackets", + label="clean", + content="See [the doc]() for details.", + files=("docs/a.md",), + ), + CorpusCase( + name="dead_link_with_title_still_flagged", + label="hallucinated", + # Title stripping must not cost recall: a dead path with a title is + # still a dead link. + content='See [the doc](docs/missing.md "Read me").', + expected=(ExpectedFinding(FindingKind.DEAD_LINK, "docs/missing.md"),), + ), ) diff --git a/tests/test_behavioral.py b/tests/test_behavioral.py index a4fe066..18d95ed 100644 --- a/tests/test_behavioral.py +++ b/tests/test_behavioral.py @@ -88,17 +88,18 @@ def test_semantic_disabled_does_not_invoke_judge(): def test_semantic_faithful_produces_no_findings_and_sets_flag(): judge = _FakeJudge(SemanticVerdict(faithful=True)) - ctx = VerifyContext(judge=judge, semantic=True) + ctx = VerifyContext(judge=judge, semantic=True, passages="the source passage") result = verify("the generated answer", ctx) assert result.semantic_ran is True assert [f for f in result.findings if f.kind is FindingKind.SEMANTIC] == [] - # The content is forwarded as the answer. + # The content is the answer; the declared passages are the ground truth. assert judge.calls[0]["answer"] == "the generated answer" + assert judge.calls[0]["passages"] == "the source passage" def test_semantic_unfaithful_emits_one_error_per_issue(): judge = _FakeJudge(SemanticVerdict(faithful=False, issues=["claim A", "claim B"])) - ctx = VerifyContext(judge=judge, semantic=True) + ctx = VerifyContext(judge=judge, semantic=True, passages="the source passage") result = verify("content", ctx) sem = [f for f in result.findings if f.kind is FindingKind.SEMANTIC] assert [f.detail for f in sem] == ["claim A", "claim B"] @@ -112,15 +113,46 @@ def test_semantic_requested_without_judge_warns(): sem = [f for f in result.findings if f.kind is FindingKind.SEMANTIC] assert len(sem) == 1 assert sem[0].severity == "warning" + assert "no judge" in sem[0].detail assert result.semantic_ran is False +def test_semantic_judge_failing_protocol_check_names_the_judge(): + # A judge that IS provided but lacks score() must not be reported as + # "no judge was provided" — the message names the failing object. + class _NotAJudge: + pass + + ctx = VerifyContext(judge=_NotAJudge(), semantic=True, passages="src") + result = verify("content", ctx) + sem = [f for f in result.findings if f.kind is FindingKind.SEMANTIC] + assert len(sem) == 1 + assert sem[0].severity == "warning" + assert "does not satisfy the Judge protocol" in sem[0].detail + assert "_NotAJudge" in sem[0].detail + assert result.semantic_ran is False + + +def test_semantic_without_passages_warns_and_skips_judge(): + # Judging content against itself is vacuous — with no passages the + # judge must not be called at all. + judge = _FakeJudge(SemanticVerdict(faithful=True)) + ctx = VerifyContext(judge=judge, semantic=True) # no passages + result = verify("content", ctx) + sem = [f for f in result.findings if f.kind is FindingKind.SEMANTIC] + assert len(sem) == 1 + assert sem[0].severity == "warning" + assert "passages" in sem[0].detail + assert result.semantic_ran is False + assert judge.calls == [] + + def test_semantic_judge_raising_degrades_to_warning(): class _Raising: def score(self, query, answer, passages): raise RuntimeError("judge down") - ctx = VerifyContext(judge=_Raising(), semantic=True) + ctx = VerifyContext(judge=_Raising(), semantic=True, passages="src") result = verify("content", ctx) sem = [f for f in result.findings if f.kind is FindingKind.SEMANTIC] assert len(sem) == 1 diff --git a/tests/test_regression_author_351.py b/tests/test_regression_author_351.py index 4e00be0..2eb5893 100644 --- a/tests/test_regression_author_351.py +++ b/tests/test_regression_author_351.py @@ -143,8 +143,12 @@ def test_semantic_only_shapes_are_semantic_layer_territory(): ) assert verify(semantic_only, ctx_no_judge).ok is True - # With a judge, both shapes surface as SEMANTIC errors. - ctx = VerifyContext(judge=_StubJudge(), semantic=True) + # With a judge and source passages, both shapes surface as SEMANTIC errors. + ctx = VerifyContext( + judge=_StubJudge(), + semantic=True, + passages="The worker accepts jobs at POST /jobs. Never bind examples to 0.0.0.0.", + ) result = verify(semantic_only, ctx) assert result.semantic_ran is True assert result.ok is False diff --git a/tests/test_semantic.py b/tests/test_semantic.py index a54f2fd..7188f1f 100644 --- a/tests/test_semantic.py +++ b/tests/test_semantic.py @@ -22,6 +22,7 @@ def test_semantic_finding_when_not_faithful(): ctx = VerifyContext( semantic=True, judge=FakeJudge(faithful=False, issues=["insecure example: host=0.0.0.0"]), + passages="Bind the server to a loopback address in examples.", ) result = verify("Some content with host=0.0.0.0", ctx) semantic = [f for f in result.findings if f.kind == FindingKind.SEMANTIC]