From 0562094f8cb2bab441bed84467185382ace9fbf0 Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 17 Jul 2026 20:23:46 -0400 Subject: [PATCH] =?UTF-8?q?fix:=20review=20follow-up=20=E2=80=94=20rag=20a?= =?UTF-8?q?dapter=20crash,=20count/link=20false=20positives,=20hygiene?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remaining findings from the library review: - rag adapter read result.is_faithful, a field FaithfulnessResult has never had (verified against installed attune-rag and its current source), so the semantic layer crashed and degraded to a warning on every call. faithful is now derived from unsupported_claims; calling from inside a running event loop raises a clear error instead of asyncio.run's generic one. - Count checker: values 1900-2099 near a source keyword are treated as years unless the keyword directly follows the number, and keyword matching now requires a leading word boundary ("test" no longer matches "latest"). check_counts docstring aligned with actual skip behavior. - Link checker: targets escaping project_root (../ traversal) yield a warning instead of silently passing when the file exists elsewhere on disk; site-absolute targets (/docs/page.md) resolve under project_root. - FindingKind.CHECKER_ERROR replaces the repurposed UNRESOLVED_IMPORT for checker infrastructure failures. - Import resolution cached per call (one subprocess per distinct module). - VerifyContext.judge typed Optional[Judge]; README status unstaled; black enforced in CI and repo formatted. Co-Authored-By: Claude Fable 5 --- .github/workflows/tests.yml | 4 +- CHANGELOG.md | 28 ++++++++ README.md | 5 +- src/attune_verify/_verify.py | 64 ++++++++++-------- src/attune_verify/checkers/counts.py | 51 ++++++++++++--- src/attune_verify/checkers/imports.py | 9 ++- src/attune_verify/checkers/links.py | 60 +++++++++++------ src/attune_verify/context.py | 12 ++-- src/attune_verify/result.py | 5 +- src/attune_verify/semantic/protocol.py | 6 +- src/attune_verify/semantic/rag_adapter.py | 45 +++++++++---- tests/corpus/cases.py | 17 +++++ tests/test_behavioral.py | 79 +++++++++++++++++++++++ tests/test_semantic.py | 27 ++++++++ 14 files changed, 336 insertions(+), 76 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 36b121b..d13db9a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,6 +32,8 @@ jobs: python -m pip install --upgrade pip pip install -e ".[dev]" - name: Lint - run: ruff check src tests + run: | + ruff check src tests + black --check src tests - name: Test (with coverage + corpus precision/recall gate) run: pytest --cov=attune_verify --cov-report=term-missing diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a45095..914af80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **rag adapter no longer crashes on every call.** It read + `result.is_faithful`, a field attune-rag's `FaithfulnessResult` has never + had (its verdict is score/claims-based), so the semantic layer always + degraded to a warning. `faithful` is now derived from + `unsupported_claims`; the adapter also raises a clear error when called + from inside a running event loop instead of asyncio.run's generic one. +- **Years near a count keyword no longer false-positive.** "Released 2026 + versions of the widgets" flagged 2026 against the `widgets` source. Values + in 1900–2099 are now compared only when the keyword directly follows the + number ("2026 widgets" is still checked). +- **Count-source keywords match on a word boundary.** "test" no longer + matches inside "latest" (plural drift still matches: "widget" ~ "widgets"). +- **Link targets can no longer escape `project_root`.** `../`-traversal that + happens to hit a real file outside the root previously passed silently; it + now yields a warning (unverifiable as a project link). Site-absolute + targets (`/docs/page.md`) are resolved under `project_root` instead of the + filesystem root. +- `check_counts` docstring claimed unmatched claims yield warnings; they are + (and were) silently skipped — the docstring now says so. - **Fences with an info string are now extracted.** ` ```python title="ex.py" ` previously matched nothing, so everything inside the fence went unchecked — a silent false negative for any MkDocs/Docusaurus-style generated doc. @@ -23,6 +42,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 checker-level warning; same for a bad `env_python` in the import checker. Both now degrade per-flag / per-import (warning) and keep checking the rest. +### Added + +- **`FindingKind.CHECKER_ERROR`** — checker infrastructure failures carry + their own kind instead of repurposing `UNRESOLVED_IMPORT`. +- Import resolution is cached per `verify()` call — repeated imports of the + same module across fences no longer re-launch the interpreter. +- `VerifyContext.judge` is typed `Optional[Judge]` (was `object`). +- CI now enforces `black --check` alongside ruff; the codebase is formatted. + ## [0.2.1] - 2026-06-22 Verifier-accuracy fixes — `verify()` now catches three hallucination diff --git a/README.md b/README.md index ada9b3a..583a359 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,10 @@ verify checks *"does this named thing exist?"* ## Status -Pre-alpha — spec complete, implementation in progress. +Alpha — the deterministic core (imports, flags, links, counts) is shipped +and guarded by a labeled precision/recall corpus (gated ≥ 0.95 each) and +mutation testing (gated ≥ 0.75). The LLM semantic layer is optional via +the `[rag]` extra. ## License diff --git a/src/attune_verify/_verify.py b/src/attune_verify/_verify.py index 13fb73a..980e74e 100644 --- a/src/attune_verify/_verify.py +++ b/src/attune_verify/_verify.py @@ -1,4 +1,5 @@ """Core verify() orchestration — runs all checkers and the semantic layer.""" + from __future__ import annotations import logging @@ -54,6 +55,7 @@ def verify(content: str, context: VerifyContext) -> VerifyResult: # Per-checker wrappers — each catches exceptions and surfaces as a warning # --------------------------------------------------------------------------- + def _check_imports(content: str, context: VerifyContext) -> List[Finding]: fences = extract_code_fences(content) return check_imports(fences, env_python=context.env_python) @@ -92,12 +94,14 @@ def _run_checker( except Exception as exc: # noqa: BLE001 # INTENTIONAL: individual checker failures must not abort the run. logger.exception("checker '%s' raised: %s", name, exc) - result.findings.append(Finding( - kind=FindingKind.UNRESOLVED_IMPORT, # closest kind for infra error - detail=f"Checker '{name}' failed: {exc}", - evidence="", - severity="warning", - )) + result.findings.append( + Finding( + kind=FindingKind.CHECKER_ERROR, + detail=f"Checker '{name}' failed: {exc}", + evidence="", + severity="warning", + ) + ) def _run_semantic(result: VerifyResult, content: str, context: VerifyContext) -> None: @@ -105,15 +109,17 @@ def _run_semantic(result: VerifyResult, content: str, context: VerifyContext) -> from attune_verify.semantic.protocol import Judge # noqa: PLC0415 if context.judge is None or not isinstance(context.judge, Judge): - result.findings.append(Finding( - kind=FindingKind.SEMANTIC, - detail=( - "Semantic layer requested (context.semantic=True) " - "but no judge was provided in VerifyContext.judge" - ), - evidence="", - severity="warning", - )) + result.findings.append( + Finding( + kind=FindingKind.SEMANTIC, + detail=( + "Semantic layer requested (context.semantic=True) " + "but no judge was provided in VerifyContext.judge" + ), + evidence="", + severity="warning", + ) + ) return try: @@ -125,18 +131,22 @@ def _run_semantic(result: VerifyResult, content: str, context: VerifyContext) -> result.semantic_ran = True if not verdict.faithful: for issue in verdict.issues: - result.findings.append(Finding( - kind=FindingKind.SEMANTIC, - detail=issue, - evidence="", - severity="error", - )) + result.findings.append( + Finding( + kind=FindingKind.SEMANTIC, + detail=issue, + evidence="", + severity="error", + ) + ) except Exception as exc: # noqa: BLE001 # INTENTIONAL: semantic layer is opt-in; failures degrade gracefully. logger.exception("semantic judge raised: %s", exc) - result.findings.append(Finding( - kind=FindingKind.SEMANTIC, - detail=f"Semantic judge failed: {exc}", - evidence="", - severity="warning", - )) + result.findings.append( + Finding( + kind=FindingKind.SEMANTIC, + detail=f"Semantic judge failed: {exc}", + evidence="", + severity="warning", + ) + ) diff --git a/src/attune_verify/checkers/counts.py b/src/attune_verify/checkers/counts.py index 930ad45..b053aa7 100644 --- a/src/attune_verify/checkers/counts.py +++ b/src/attune_verify/checkers/counts.py @@ -2,11 +2,16 @@ from __future__ import annotations +import re from typing import Callable, Dict, List, Union from attune_verify._extract import NumericClaim from attune_verify.result import Finding, FindingKind +# Values in this range near a source keyword are far more likely to be years +# than counts ("Released 2026 versions of the widgets" is not a widget count). +_YEAR_MIN, _YEAR_MAX = 1900, 2099 + def check_counts( claims: List[NumericClaim], @@ -14,10 +19,13 @@ def check_counts( ) -> List[Finding]: """Verify numeric claims match count_sources values. - Counts cannot be inferred — the caller must supply them. Any numeric - claim in the content is matched against count_sources by value. Claims - with no matching source entry are flagged as warnings (unverifiable), - not errors. + Counts cannot be inferred — the caller must supply them. Each numeric + claim is matched to the source its surrounding text names and compared + against that source's value. Claims whose context names no source are + silently skipped (unverifiable without a source, and flagging every + stray number would be noise). Year-like values (1900–2099) are compared + only when the source keyword directly follows the number ("2026 widgets"), + so dates near a keyword don't false-positive. Args: claims: Numeric claims extracted from generated content. @@ -25,7 +33,7 @@ def check_counts( Values may be plain ints or zero-argument callables. Returns: - List of findings for mismatched or unverifiable counts. + List of findings for mismatched counts. """ if not count_sources: return [] @@ -43,7 +51,11 @@ def check_counts( # unrelated source (e.g. "12 tests" passing because some other source # also equals 12) — cross-contamination. close_label = _find_close_label(claim.context, resolved_sources) - if close_label is not None and claim.value != resolved_sources[close_label]: + if close_label is None: + continue + if _year_like(claim.value) and not _label_follows_number(claim, close_label): + continue + if claim.value != resolved_sources[close_label]: expected = resolved_sources[close_label] findings.append( Finding( @@ -64,10 +76,33 @@ def _find_close_label( context: str, sources: Dict[str, int], ) -> str | None: - """Find a source label whose keywords appear in the claim's context.""" + """Find a source label whose keywords appear in the claim's context. + + Keywords match on a leading word boundary — "test" matches "tests" but + not "latest" — a bare substring test false-matched inside longer words. + """ context_lower = context.lower() for label in sources: words = label.lower().split() - if any(w in context_lower for w in words if len(w) > 3): + if any(re.search(rf"\b{re.escape(w)}", context_lower) for w in words if len(w) > 3): return label return None + + +def _year_like(value: int) -> bool: + return _YEAR_MIN <= value <= _YEAR_MAX + + +def _label_follows_number(claim: NumericClaim, label: str) -> bool: + """True when a label keyword is one of the two tokens after the number. + + "2026 widgets" reads as a widget count; "2026 versions of the widgets" + reads as a year that merely has the keyword nearby. + """ + match = re.search(rf"\b{claim.value}\b((?:\s+\S+){{1,2}})", claim.context.lower()) + if match is None: + return False + following = match.group(1) + return any( + re.search(rf"\b{re.escape(w)}", following) for w in label.lower().split() if len(w) > 3 + ) diff --git a/src/attune_verify/checkers/imports.py b/src/attune_verify/checkers/imports.py index d80304e..7b6863b 100644 --- a/src/attune_verify/checkers/imports.py +++ b/src/attune_verify/checkers/imports.py @@ -29,6 +29,9 @@ def check_imports( List of findings for unresolvable imports. """ findings: List[Finding] = [] + # Each resolution is a subprocess; repeated imports of the same module + # across fences are common, so resolve each module once per call. + resolution_cache: dict[str, bool] = {} for fence in fences: # "" is a bare fence: LLM output routinely omits the language tag, so # parse speculatively — non-Python content fails ast.parse and is skipped. @@ -42,7 +45,11 @@ def check_imports( for module in _modules_from_node(node): location = f"line {fence.line}" if fence.line else None try: - resolved = _resolves(module, env_python) + if module in resolution_cache: + resolved = resolution_cache[module] + else: + resolved = _resolves(module, env_python) + resolution_cache[module] = resolved except (OSError, subprocess.TimeoutExpired) as exc: # Resolution infrastructure failed (bad env_python, timeout). # Degrade per-import — the remaining imports must still run. diff --git a/src/attune_verify/checkers/links.py b/src/attune_verify/checkers/links.py index a2fdd72..f061d33 100644 --- a/src/attune_verify/checkers/links.py +++ b/src/attune_verify/checkers/links.py @@ -1,4 +1,5 @@ """Link checker — verifies markdown link targets resolve to real files.""" + from __future__ import annotations from pathlib import Path @@ -35,24 +36,47 @@ def check_links( if not path_part: continue if project_root is None: - findings.append(Finding( - kind=FindingKind.DEAD_LINK, - detail=( - f"Link '{target}' cannot be verified " - "(no project_root in VerifyContext)" - ), - evidence=f"[{link.text}]({link.target})", - location=f"line {link.line}" if link.line else None, - severity="warning", - )) + findings.append( + Finding( + kind=FindingKind.DEAD_LINK, + detail=( + f"Link '{target}' cannot be verified " "(no project_root in VerifyContext)" + ), + evidence=f"[{link.text}]({link.target})", + location=f"line {link.line}" if link.line else None, + severity="warning", + ) + ) + continue + root = project_root.resolve() + # Site-absolute targets (/docs/page.md) mean root-relative in generated + # docs; joining them raw would make Path use the filesystem root. + rel = path_part.lstrip("/") if path_part.startswith("/") else path_part + resolved = (root / rel).resolve() + if not resolved.is_relative_to(root): + # ../-traversal out of the declared truth boundary: the file may + # exist on disk, but it cannot be verified AS a project link. + # Warning, not error — same "never a silent pass" rule as flags. + findings.append( + Finding( + kind=FindingKind.DEAD_LINK, + detail=( + f"Link '{target}' resolves outside project_root " "and cannot be verified" + ), + evidence=f"[{link.text}]({link.target})", + location=f"line {link.line}" if link.line else None, + severity="warning", + ) + ) continue - resolved = (project_root / path_part).resolve() if not resolved.exists(): - findings.append(Finding( - kind=FindingKind.DEAD_LINK, - detail=f"Link target '{path_part}' does not exist", - evidence=f"[{link.text}]({link.target})", - location=f"line {link.line}" if link.line else None, - severity="error", - )) + findings.append( + Finding( + kind=FindingKind.DEAD_LINK, + detail=f"Link target '{path_part}' does not exist", + evidence=f"[{link.text}]({link.target})", + location=f"line {link.line}" if link.line else None, + severity="error", + ) + ) return findings diff --git a/src/attune_verify/context.py b/src/attune_verify/context.py index 38adcfd..87856af 100644 --- a/src/attune_verify/context.py +++ b/src/attune_verify/context.py @@ -2,12 +2,16 @@ The caller declares WHERE truth comes from; verify performs the lookups. """ + from __future__ import annotations import sys from dataclasses import dataclass, field from pathlib import Path -from typing import Callable, Dict, FrozenSet, Optional, Union +from typing import TYPE_CHECKING, Callable, Dict, FrozenSet, Optional, Union + +if TYPE_CHECKING: + from attune_verify.semantic.protocol import Judge @dataclass @@ -39,8 +43,6 @@ class VerifyContext: env_python: str = field(default_factory=lambda: sys.executable) help_commands: Dict[str, str] = field(default_factory=dict) allowed_help_cmds: FrozenSet[str] = field(default_factory=frozenset) - count_sources: Dict[str, Union[int, Callable[[], int]]] = field( - default_factory=dict - ) - judge: object = None # Judge protocol instance + count_sources: Dict[str, Union[int, Callable[[], int]]] = field(default_factory=dict) + judge: Optional["Judge"] = None semantic: bool = False diff --git a/src/attune_verify/result.py b/src/attune_verify/result.py index f9e4434..3519f62 100644 --- a/src/attune_verify/result.py +++ b/src/attune_verify/result.py @@ -2,6 +2,7 @@ Public: FindingKind, Finding, VerifyResult, VerificationError, raise_if_failed """ + from __future__ import annotations from dataclasses import dataclass, field @@ -10,13 +11,15 @@ class FindingKind(str, Enum): - """Typed kinds matching the four verification modes.""" + """Typed kinds: the four verification modes, the semantic layer, and + CHECKER_ERROR for infrastructure failures inside a checker itself.""" UNRESOLVED_IMPORT = "unresolved_import" UNKNOWN_FLAG = "unknown_flag" DEAD_LINK = "dead_link" COUNT_MISMATCH = "count_mismatch" SEMANTIC = "semantic" + CHECKER_ERROR = "checker_error" @dataclass(frozen=True) diff --git a/src/attune_verify/semantic/protocol.py b/src/attune_verify/semantic/protocol.py index 51cf9af..47ba6e0 100644 --- a/src/attune_verify/semantic/protocol.py +++ b/src/attune_verify/semantic/protocol.py @@ -3,6 +3,7 @@ The core library never imports attune-rag directly. Any object satisfying Judge can be injected: the built-in rag adapter, a skill-judge, or a fake. """ + from __future__ import annotations from dataclasses import dataclass, field @@ -22,8 +23,9 @@ class SemanticVerdict: class Judge(Protocol): """Protocol that any semantic judge must satisfy. - Signature matches attune-rag's FaithfulnessJudge.score() so the - rag adapter is near-trivial (verified against rag 0.2.0). + The signature mirrors attune-rag's FaithfulnessJudge.score(), but the + return type differs: FaithfulnessResult has no boolean verdict, so the + rag adapter derives ``faithful`` from its unsupported_claims. """ def score( diff --git a/src/attune_verify/semantic/rag_adapter.py b/src/attune_verify/semantic/rag_adapter.py index 6019c53..fe817fa 100644 --- a/src/attune_verify/semantic/rag_adapter.py +++ b/src/attune_verify/semantic/rag_adapter.py @@ -2,11 +2,8 @@ Only imported when the [rag] extra is installed. Provides make_rag_judge() which wraps attune-rag's FaithfulnessJudge. - -Phase-3 verification item: confirm FaithfulnessResult field names -(is_faithful / unsupported_claims) against installed attune-rag before -wiring this adapter. """ + from __future__ import annotations import asyncio @@ -18,9 +15,28 @@ from attune_verify.semantic.protocol import Judge +def _to_verdict(result: object) -> SemanticVerdict: + """Translate attune-rag's FaithfulnessResult into a SemanticVerdict. + + FaithfulnessResult carries no boolean verdict — its ``score`` is + ``supported / (supported + unsupported)`` — so "faithful" here means + no unsupported claims were found. + """ + unsupported = list(getattr(result, "unsupported_claims", None) or []) + return SemanticVerdict( + faithful=not unsupported, + issues=unsupported, + raw=result, + ) + + def make_rag_judge(**kwargs: object) -> "Judge": """Build a Judge wrapping attune-rag's FaithfulnessJudge. + The returned Judge is synchronous: FaithfulnessJudge.score() is async + and is driven with asyncio.run(), so it must be called from sync code + (or a worker thread) — never from inside a running event loop. + Args: **kwargs: Passed directly to FaithfulnessJudge.__init__ (api_key, model, timeout, etc.). @@ -48,14 +64,19 @@ def score( answer: str, passages: Union[str, List[str]], ) -> SemanticVerdict: - # FaithfulnessJudge.score() is async — run synchronously here. - # TODO(Phase-3): verify is_faithful / unsupported_claims field names - # against installed attune-rag before relying on them. + try: + asyncio.get_running_loop() + except RuntimeError: + pass # no running loop — safe to drive the coroutine + else: + # asyncio.run() inside a running loop raises anyway, but with + # a message that doesn't say what to do about it. + raise RuntimeError( + "make_rag_judge()'s Judge is synchronous and cannot be " + "called from inside a running event loop; call verify() " + "from sync code or run it in a worker thread." + ) result = asyncio.run(inner.score(query, answer, passages)) - return SemanticVerdict( - faithful=result.is_faithful, # type: ignore[attr-defined] - issues=result.unsupported_claims or [], # type: ignore[attr-defined] - raw=result, - ) + return _to_verdict(result) return _Adapter() # type: ignore[return-value] diff --git a/tests/corpus/cases.py b/tests/corpus/cases.py index dc3d0fa..7443053 100644 --- a/tests/corpus/cases.py +++ b/tests/corpus/cases.py @@ -128,6 +128,23 @@ def _py(code: str) -> str: help_commands={"mytool": "Options:\n --verbose Be loud\n --help Show help\n"}, expected=(ExpectedFinding(FindingKind.UNKNOWN_FLAG, "--nonexistent"),), ), + CorpusCase( + name="clean_year_near_count_keyword", + label="clean", + # 2026 is a year, not a widget count — a keyword merely nearby must + # not turn it into a COUNT_MISMATCH (regression for a false positive). + content="Released 2026 versions of the widgets.", + count_sources={"widgets": 12}, + ), + CorpusCase( + name="year_valued_count_still_checked", + label="hallucinated", + # A year-like value directly before the keyword IS a count claim and + # must still be compared — the year guard must not cost recall here. + content="There are 2026 widgets in stock.", + count_sources={"widgets": 12}, + expected=(ExpectedFinding(FindingKind.COUNT_MISMATCH, "2026"),), + ), # --------------------------------------------------------------- evasion CorpusCase( name="evasion_multi_import", diff --git a/tests/test_behavioral.py b/tests/test_behavioral.py index 3577cba..a4fe066 100644 --- a/tests/test_behavioral.py +++ b/tests/test_behavioral.py @@ -52,6 +52,8 @@ def boom(content, context): assert len(infra) == 1 assert "kaboom" in infra[0].detail assert infra[0].severity == "warning" + # Infra failures carry their own kind — not a repurposed content kind. + assert infra[0].kind is FindingKind.CHECKER_ERROR # The failed checker is NOT recorded as checked, but the others are. assert "imports" not in result.checked assert {"flags", "links", "counts"} <= set(result.checked) @@ -161,6 +163,33 @@ def test_links_dead_file_is_error(tmp_path): assert "missing.md" in findings[0].detail +def test_links_traversal_outside_root_is_warning_even_if_file_exists(tmp_path): + # ../-escapes can hit a real file on disk (e.g. /etc/passwd) — that must + # not read as a verified project link. Unverifiable -> warning, never + # a silent pass. + outside = tmp_path / "outside.md" + outside.write_text("x", encoding="utf-8") + root = tmp_path / "project" + root.mkdir() + links = [MarkdownLink(text="up", target="../outside.md", line=1)] + findings = check_links(links, project_root=root) + assert len(findings) == 1 + assert findings[0].severity == "warning" + assert "outside project_root" in findings[0].detail + + +def test_links_site_absolute_target_is_root_relative(tmp_path): + # /docs/page.md means "from the project root" in generated docs — it must + # be resolved under project_root, not against the filesystem root. + (tmp_path / "docs").mkdir() + (tmp_path / "docs" / "page.md").write_text("x", encoding="utf-8") + assert check_links([MarkdownLink(text="p", target="/docs/page.md", line=1)], tmp_path) == [] + + findings = check_links([MarkdownLink(text="pw", target="/etc/passwd", line=1)], tmp_path) + assert len(findings) == 1 + assert findings[0].severity == "error" # root/etc/passwd does not exist + + # --------------------------------------------------------------------------- # Flag checker branches # --------------------------------------------------------------------------- @@ -258,6 +287,27 @@ def test_bare_fence_non_python_is_skipped(): assert check_imports(fences, env_python=sys.executable) == [] +def test_repeated_imports_resolved_once_per_call(monkeypatch): + import attune_verify.checkers.imports as imports_mod + + calls = [] + real_run = imports_mod.subprocess.run + + def counting_run(*args, **kwargs): + calls.append(args[0]) + return real_run(*args, **kwargs) + + monkeypatch.setattr(imports_mod.subprocess, "run", counting_run) + fences = [ + CodeFence(language="python", content="import os\nimport os.path\n", line=1), + CodeFence(language="python", content="import os\n", line=5), + ] + findings = check_imports(fences, env_python=sys.executable) + assert findings == [] + # os appears twice but resolves once; os.path is distinct. + assert len(calls) == 2 + + # --------------------------------------------------------------------------- # Extractors: line numbers, language defaulting, context windows # --------------------------------------------------------------------------- @@ -320,6 +370,35 @@ def test_count_source_callable_is_resolved(): assert "9" in findings[0].detail +def test_count_label_matches_on_word_boundary_only(): + # "test" must not match inside "latest" — that false-positived clean docs. + claims = [NumericClaim(value=99, context="get the latest 99 release notes", line=1)] + assert check_counts(claims, count_sources={"test": 50}) == [] + # A leading boundary still allows plural drift: "widget" matches "widgets". + claims = [NumericClaim(value=99, context="there are 99 widgets here", line=1)] + assert len(check_counts(claims, count_sources={"widget": 12})) == 1 + + +def test_year_near_label_is_skipped_unless_adjacent(): + # A year with a source keyword merely nearby is a date, not a count. + claims = [NumericClaim(value=2026, context="released 2026 versions of the widgets", line=1)] + assert check_counts(claims, count_sources={"widgets": 12}) == [] + # But "2026 widgets" IS a widget count and must still be compared. + claims = [NumericClaim(value=2026, context="there are 2026 widgets in stock", line=1)] + findings = check_counts(claims, count_sources={"widgets": 12}) + assert len(findings) == 1 + assert findings[0].severity == "error" + + +def test_year_guard_boundaries(): + # Outside 1900-2099 the window match suffices; inside it needs adjacency. + for value, expected_findings in ((1899, 1), (1900, 0), (2099, 0), (2100, 1)): + claims = [ + NumericClaim(value=value, context=f"released {value} builds of the widgets", line=1) + ] + assert len(check_counts(claims, count_sources={"widgets": 12})) == expected_findings + + def test_count_source_callable_matching_value_is_clean(): claims = [NumericClaim(value=9, context="there are 9 plugins", line=1)] assert check_counts(claims, count_sources={"plugins": lambda: 9}) == [] diff --git a/tests/test_semantic.py b/tests/test_semantic.py index dbe1284..a54f2fd 100644 --- a/tests/test_semantic.py +++ b/tests/test_semantic.py @@ -1,4 +1,5 @@ """Tests for the semantic layer (T4).""" + from attune_verify import VerifyContext, verify from attune_verify.result import FindingKind from attune_verify.semantic.protocol import Judge, SemanticVerdict @@ -43,3 +44,29 @@ def test_semantic_disabled_by_default(): result = verify("Some content.", ctx) assert result.semantic_ran is False assert not any(f.kind == FindingKind.SEMANTIC for f in result.findings) + + +class _StubFaithfulnessResult: + """Shape of attune-rag's FaithfulnessResult: score-based, no boolean.""" + + def __init__(self, unsupported: list) -> None: + self.unsupported_claims = unsupported + self.score = 1.0 if not unsupported else 0.5 + + +def test_rag_verdict_faithful_when_no_unsupported_claims(): + from attune_verify.semantic.rag_adapter import _to_verdict + + verdict = _to_verdict(_StubFaithfulnessResult([])) + assert verdict.faithful is True + assert verdict.issues == [] + + +def test_rag_verdict_unfaithful_carries_unsupported_claims(): + from attune_verify.semantic.rag_adapter import _to_verdict + + result = _StubFaithfulnessResult(["the /run route does not exist"]) + verdict = _to_verdict(result) + assert verdict.faithful is False + assert verdict.issues == ["the /run route does not exist"] + assert verdict.raw is result