From 8d32b03beed86850bc5bcd8301e464e979aba4ac Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 11 Aug 2026 00:20:45 -0400 Subject: [PATCH] =?UTF-8?q?fix:=202026-08-10=20audit=20follow-up=20?= =?UTF-8?q?=E2=80=94=20count/flag/link=20accuracy,=20semantic=20passages?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accuracy fixes from the full library audit, each with paired clean + hallucinated corpus cases: - Comma-grouped numbers ("1,234") are one claim; decimals ("94.53") and version components ("Python 3.10") are no longer claims at all. - Short count-source labels ("api", "eps") are no longer silently dead; short words match on exact word boundaries and only when the label has no longer word, so stopwords in mixed labels ("number of tests") can't drag unrelated numbers into the source. - Flags inside single-span commands (`mytool --flag`) and bash/sh/shell/ console/zsh fences are now checked — previously only a flag backticked alone was seen, despite a comment claiming otherwise. - Markdown link titles ([text](path "Title")) and targets no longer break path existence checks. - The semantic layer judges against caller-declared VerifyContext.passages instead of the content itself (vacuously faithful); with no passages the judge is skipped with a warning. Bad-judge and no-judge warnings are distinct messages. - Import resolution passes the module name via argv (defense in depth). - Corpus flag cases use the realistic single-span form. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 57 ++++++++++++ src/attune_verify/_extract.py | 38 ++++++-- src/attune_verify/_verify.py | 32 +++++-- src/attune_verify/checkers/counts.py | 39 ++++++-- src/attune_verify/checkers/flags.py | 93 ++++++++++++++------ src/attune_verify/checkers/imports.py | 11 ++- src/attune_verify/context.py | 9 +- tests/corpus/cases.py | 122 +++++++++++++++++++++++++- tests/test_behavioral.py | 40 ++++++++- tests/test_regression_author_351.py | 8 +- tests/test_semantic.py | 1 + 11 files changed, 393 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c84beba..d0e2c00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,63 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +Accuracy fixes from the 2026-08-10 library audit: three numeric-claim +false-positive classes stopped, two silent false-negative classes closed +(short count labels, single-span/fenced CLI flags), markdown link titles +handled, and the semantic layer now judges against real source passages +instead of the content itself. Every fix carries a regression corpus case. + +### Fixed + +- **Comma-grouped numbers are one claim.** "1,234 tests" previously + extracted the "234" fragment and flagged an error-severity count + mismatch against `tests=1234`; `1,234` is now extracted as the single + value 1234 (commas stripped). Grouped values also stay checked when + year-valued — "2,026 widgets" is a count, never a year. +- **Decimals and version components are not counts.** "94.53" near a + source keyword extracted 94 and 53 as separate claims and flagged both; + the "10" in "Python 3.10" was flagged against a nearby source. Digit + runs adjacent to a decimal point are now skipped. +- **Short count-source labels are no longer silently dead.** Label words + of length ≤ 3 ("api", "eps") were dropped by the keyword filter, so + those sources could never match any claim — a silent false negative. + Short words now require an exact word-boundary match on both sides + ("api" matches "api" but not "rapid"), and participate only when the + label has no longer word — in a mixed label ("number of tests") a short + word is usually a stopword, and letting "of" match ordinary prose would + drag unrelated numbers into the source. Longer words keep the + leading-boundary match so plural drift still hits. +- **Flags inside real command spans and shell fences are now checked.** + The extractor only matched a flag backticked alone (`` `--flag` ``); + the common form `` `mytool --flag` `` — the whole command in one span — + matched nothing, despite a comment claiming it was handled, and flags + inside ```` ```bash ```` fences were unchecked. Both are now extracted + (fences: bash/sh/shell/console/zsh), with the command guessed from the + span or line itself before falling back to preceding prose. +- **Markdown link titles no longer break path checks.** + `[text](docs/a.md "Read me")` treated the whole `docs/a.md "Read me"` + string as the path and errored even when the file exists. The optional + title is stripped and `` targets are unwrapped; dead + paths with titles are still flagged. +- **The semantic layer no longer judges content against itself.** + `verify()` passed the generated content as its own source passages — + vacuously faithful by construction. `VerifyContext` gains a `passages` + field that is forwarded to the judge; when `semantic=True` with no + passages, verify degrades gracefully (warning, judge not called). +- **The no-judge warning no longer misreports a bad judge.** A judge that + was provided but fails the `Judge` protocol check was reported as "no + judge was provided"; the message now names the failing object and the + protocol mismatch. + +### Changed + +- Import resolution passes the module name to the child interpreter via + `argv` instead of f-string interpolation into the `-c` program — + defense in depth (`ast` already guarantees identifier-safe names). +- Corpus flag cases use the realistic single-span form + (`` `mytool --verbose` ``) instead of the unnatural split form; the + split form remains covered by `evasion_substring_flag`. + ## [0.2.2] - 2026-07-17 Accuracy + reliability patch from a full library review: five silent diff --git a/src/attune_verify/_extract.py b/src/attune_verify/_extract.py index 568c6a1..ffe0c9e 100644 --- a/src/attune_verify/_extract.py +++ b/src/attune_verify/_extract.py @@ -42,7 +42,15 @@ class NumericClaim: re.MULTILINE | re.DOTALL, ) _LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)]+)\)") -_NUM_RE = re.compile(r"\b(\d{2,})\b") # 2+ digit numbers (skip single digits) +# 2+ digit numbers (skip single digits). Comma-grouped values ("1,234") are one +# claim — the first alternative captures the whole group before the bare \d{2,} +# can grab a fragment. Digit runs touching a decimal point ("94.53", the "10" +# in "Python 3.10") are decimal/version components, not counts, and are skipped +# via the surrounding lookarounds. +_NUM_RE = re.compile(r"(?. +_LINK_TITLE_RE = re.compile(r"""^(\S+)\s+("[^"]*"|'[^']*'|\([^)]*\))$""") def extract_code_fences(content: str) -> List[CodeFence]: @@ -63,22 +71,42 @@ def extract_code_fences(content: str) -> List[CodeFence]: def extract_links(content: str) -> List[MarkdownLink]: - """Extract all markdown links from content.""" + """Extract all markdown links from content. + + Targets are normalized: an optional markdown title + (``docs/a.md "Read me"``) is stripped and ```` wrapping is + removed, so checkers see only the path. + """ links = [] for match in _LINK_RE.finditer(content): line = content[: match.start()].count("\n") + 1 links.append( MarkdownLink( text=match.group(1), - target=match.group(2), + target=_clean_link_target(match.group(2)), line=line, ) ) return links +def _clean_link_target(raw: str) -> str: + """Strip an optional title and angle-bracket wrapping from a link target.""" + target = raw.strip() + title_match = _LINK_TITLE_RE.match(target) + if title_match: + target = title_match.group(1) + if target.startswith("<") and target.endswith(">"): + target = target[1:-1] + return target + + def extract_numeric_claims(content: str) -> List[NumericClaim]: - """Extract numeric claims (2+ digit numbers) with surrounding context.""" + """Extract numeric claims (2+ digit numbers) with surrounding context. + + Comma-grouped numbers ("1,234") are one claim with the commas stripped; + decimal and version components ("94.53", "Python 3.10") are not claims. + """ claims = [] for match in _NUM_RE.finditer(content): line = content[: match.start()].count("\n") + 1 @@ -86,7 +114,7 @@ def extract_numeric_claims(content: str) -> List[NumericClaim]: end = min(len(content), match.end() + 40) claims.append( NumericClaim( - value=int(match.group(1)), + value=int(match.group(1).replace(",", "")), context=content[start:end].replace("\n", " "), line=line, ) diff --git a/src/attune_verify/_verify.py b/src/attune_verify/_verify.py index 980e74e..448f04f 100644 --- a/src/attune_verify/_verify.py +++ b/src/attune_verify/_verify.py @@ -25,7 +25,8 @@ def verify(content: str, context: VerifyContext) -> VerifyResult: Deterministic checkers always run and are independent — a failure in one does not abort the others. The semantic layer runs only when - context.semantic is True and a judge is available. + context.semantic is True and both a judge and source passages are + available. Args: content: LLM-generated content to verify. @@ -108,14 +109,31 @@ def _run_semantic(result: VerifyResult, content: str, context: VerifyContext) -> """Run the semantic layer if a judge is available.""" from attune_verify.semantic.protocol import Judge # noqa: PLC0415 - if context.judge is None or not isinstance(context.judge, Judge): + if context.judge is None: + detail = ( + "Semantic layer requested (context.semantic=True) " + "but no judge was provided in VerifyContext.judge" + ) + elif not isinstance(context.judge, Judge): + detail = ( + "Semantic layer requested (context.semantic=True) but the " + f"provided judge ({type(context.judge).__name__}) does not " + "satisfy the Judge protocol (missing a compatible score())" + ) + elif not context.passages: + # Without independent source passages the judge would score the + # content against itself — vacuously faithful, so skip instead. + detail = ( + "Semantic layer requested (context.semantic=True) but no " + "source passages were provided in VerifyContext.passages" + ) + else: + detail = None + if detail is not None: result.findings.append( Finding( kind=FindingKind.SEMANTIC, - detail=( - "Semantic layer requested (context.semantic=True) " - "but no judge was provided in VerifyContext.judge" - ), + detail=detail, evidence="", severity="warning", ) @@ -126,7 +144,7 @@ def _run_semantic(result: VerifyResult, content: str, context: VerifyContext) -> verdict = context.judge.score( query="Verify this generated content for faithfulness", answer=content, - passages=content, + passages=context.passages, ) result.semantic_ran = True if not verdict.faithful: diff --git a/src/attune_verify/checkers/counts.py b/src/attune_verify/checkers/counts.py index b053aa7..2d83fdc 100644 --- a/src/attune_verify/checkers/counts.py +++ b/src/attune_verify/checkers/counts.py @@ -83,12 +83,38 @@ def _find_close_label( """ context_lower = context.lower() for label in sources: - words = label.lower().split() - if any(re.search(rf"\b{re.escape(w)}", context_lower) for w in words if len(w) > 3): + if any(_word_matches(w, context_lower) for w in _match_words(label)): return label return None +def _match_words(label: str) -> list[str]: + """Return the label words eligible for keyword matching. + + Short words (<= 3 chars) participate only when the label has no longer + word to match on. A label like "api" was silently dead when short words + were dropped outright; but in a mixed label like "number of tests" a + short word is usually a stopword — letting "of" match ordinary prose + would drag unrelated numbers into the source. + """ + words = label.lower().split() + long_words = [w for w in words if len(w) > 3] + return long_words if long_words else words + + +def _word_matches(word: str, text: str) -> bool: + """Match a label keyword in text, calibrated to the keyword's length. + + Long words match on a leading boundary only, so plural drift still hits + ("test" ~ "tests"). Short words ("api", "eps") prefix-match far too + loosely, so they require an exact word-boundary match on both sides — + previously they were dropped entirely, silently killing their source. + """ + if len(word) > 3: + return re.search(rf"\b{re.escape(word)}", text) is not None + return re.search(rf"\b{re.escape(word)}\b", text) is not None + + def _year_like(value: int) -> bool: return _YEAR_MIN <= value <= _YEAR_MAX @@ -99,10 +125,11 @@ def _label_follows_number(claim: NumericClaim, label: str) -> bool: "2026 widgets" reads as a widget count; "2026 versions of the widgets" reads as a year that merely has the keyword nearby. """ - match = re.search(rf"\b{claim.value}\b((?:\s+\S+){{1,2}})", claim.context.lower()) + # The claim value has commas stripped, but the context still shows the + # written form — match either ("2,026 widgets" is a count, never a year). + grouped = re.escape(f"{claim.value:,}") + match = re.search(rf"\b(?:{claim.value}|{grouped})\b((?:\s+\S+){{1,2}})", claim.context.lower()) if match is None: return False following = match.group(1) - return any( - re.search(rf"\b{re.escape(w)}", following) for w in label.lower().split() if len(w) > 3 - ) + return any(_word_matches(w, following) for w in _match_words(label)) diff --git a/src/attune_verify/checkers/flags.py b/src/attune_verify/checkers/flags.py index 3651c79..ee9bfc4 100644 --- a/src/attune_verify/checkers/flags.py +++ b/src/attune_verify/checkers/flags.py @@ -4,11 +4,18 @@ import re import subprocess -from typing import Dict, FrozenSet, List +from typing import Dict, FrozenSet, List, Optional +from attune_verify._extract import _FENCE_RE, extract_code_fences from attune_verify.result import Finding, FindingKind -_FLAG_RE = re.compile(r"`(--[\w-]+)`") +# One inline code span (`mytool --flag`); fences are handled separately. +_INLINE_CODE_RE = re.compile(r"`([^`\n]+)`") +# A flag token anywhere in code text. Stops before "=value"; the negative +# lookbehind keeps it from matching the tail of a longer flag or a "---" rule. +_FLAG_TOKEN_RE = re.compile(r"(? Optional[Finding]: + """Check one flag against its command's help; None when it verifies.""" + help_text = _get_help(cmd, help_commands, allowed_help_cmds) + if help_text is None: + return Finding( + kind=FindingKind.UNKNOWN_FLAG, + detail=( + f"Flag '{flag}' could not be verified " + f"(no --help output available for command '{cmd}')" + ), + evidence=evidence, + severity="warning", + ) + if not _flag_in_help(flag, help_text): + return Finding( + kind=FindingKind.UNKNOWN_FLAG, + detail=f"Flag '{flag}' not found in '{cmd} --help'", + evidence=evidence, + severity="error", + ) + return None + + def _flag_in_help(flag: str, help_text: str) -> bool: """Return True if flag appears in help as a whole token. diff --git a/src/attune_verify/checkers/imports.py b/src/attune_verify/checkers/imports.py index 7b6863b..88fc4b8 100644 --- a/src/attune_verify/checkers/imports.py +++ b/src/attune_verify/checkers/imports.py @@ -105,12 +105,19 @@ def _modules_from_node(node: ast.AST) -> list[str]: def _resolves(module: str, env_python: str) -> bool: - """Return True if module is importable in env_python.""" + """Return True if module is importable in env_python. + + The module name travels via argv, not f-string interpolation into the + -c program — defense in depth (ast already guarantees identifier-safe + names, but the child program must not depend on that). + """ result = subprocess.run( [ env_python, "-c", - f"import importlib.util; " f"print(importlib.util.find_spec('{module}') is not None)", + "import importlib.util, sys; " + "print(importlib.util.find_spec(sys.argv[1]) is not None)", + module, ], capture_output=True, text=True, diff --git a/src/attune_verify/context.py b/src/attune_verify/context.py index 87856af..c437022 100644 --- a/src/attune_verify/context.py +++ b/src/attune_verify/context.py @@ -8,7 +8,7 @@ import sys from dataclasses import dataclass, field from pathlib import Path -from typing import TYPE_CHECKING, Callable, Dict, FrozenSet, Optional, Union +from typing import TYPE_CHECKING, Callable, Dict, FrozenSet, List, Optional, Union if TYPE_CHECKING: from attune_verify.semantic.protocol import Judge @@ -36,7 +36,11 @@ class VerifyContext: judge: Optional semantic judge implementing the Judge protocol. Required for the semantic layer; when absent and semantic=True, verify degrades gracefully (warning, not error). - semantic: Enable the LLM semantic layer. Requires a judge. + passages: Source-of-truth passage(s) the semantic judge grounds the + content against. Required for the semantic layer — judging content + against itself is vacuous, so when absent and semantic=True, + verify degrades gracefully (warning, not error). + semantic: Enable the LLM semantic layer. Requires a judge and passages. """ project_root: Optional[Path] = None @@ -45,4 +49,5 @@ class VerifyContext: allowed_help_cmds: FrozenSet[str] = field(default_factory=frozenset) count_sources: Dict[str, Union[int, Callable[[], int]]] = field(default_factory=dict) judge: Optional["Judge"] = None + passages: Optional[Union[str, List[str]]] = None semantic: bool = False diff --git a/tests/corpus/cases.py b/tests/corpus/cases.py index 7443053..14c542e 100644 --- a/tests/corpus/cases.py +++ b/tests/corpus/cases.py @@ -91,7 +91,14 @@ def _py(code: str) -> str: CorpusCase( name="clean_flag", label="clean", - content="Run `mytool` `--verbose` for detailed output.", + # Realistic form: the whole command lives in ONE backtick span. + content="Run `mytool --verbose` for detailed output.", + help_commands={"mytool": "Options:\n --verbose Be loud\n --help Show help\n"}, + ), + CorpusCase( + name="clean_flag_bash_fence", + label="clean", + content="Run it:\n```bash\n$ mytool --verbose\n```\n", help_commands={"mytool": "Options:\n --verbose Be loud\n --help Show help\n"}, ), # --------------------------------------------------------- hallucinated @@ -124,7 +131,10 @@ def _py(code: str) -> str: CorpusCase( name="fake_flag", label="hallucinated", - content="Pass `mytool` `--nonexistent` to enable it.", + # Realistic form: the whole command lives in ONE backtick span — the + # v0.2.2 extractor only matched a flag backticked alone, so this + # (the most common way LLMs write commands) passed silently. + content="Pass `mytool --nonexistent` to enable it.", help_commands={"mytool": "Options:\n --verbose Be loud\n --help Show help\n"}, expected=(ExpectedFinding(FindingKind.UNKNOWN_FLAG, "--nonexistent"),), ), @@ -195,4 +205,112 @@ def _py(code: str) -> str: count_sources={"tests": 50, "modules": 12}, expected=(ExpectedFinding(FindingKind.COUNT_MISMATCH, "12"),), ), + # ------------------------------------------------- 2026-08-10 audit fixes + CorpusCase( + name="clean_comma_grouped_count", + label="clean", + # "1,234" is ONE number. The v0.2.2 extractor grabbed the "234" + # fragment and flagged it against tests=1234 (error false positive). + content="The suite runs 1,234 tests on every push.", + count_sources={"tests": 1234}, + ), + CorpusCase( + name="comma_grouped_count_mismatch", + label="hallucinated", + # Comma handling must not cost recall: a wrong grouped count is + # still a mismatch. + content="The suite runs 1,234 tests on every push.", + count_sources={"tests": 999}, + expected=(ExpectedFinding(FindingKind.COUNT_MISMATCH, "1234"),), + ), + CorpusCase( + name="comma_grouped_year_valued_count", + label="hallucinated", + # "2,026" is year-VALUED but comma-grouped — nobody writes a year + # with a thousands separator, so it is a count and must be checked. + content="There are 2,026 widgets in stock.", + count_sources={"widgets": 12}, + expected=(ExpectedFinding(FindingKind.COUNT_MISMATCH, "2026"),), + ), + CorpusCase( + name="clean_decimal_near_keyword", + label="clean", + # "94.53" is a decimal, not two counts — the v0.2.2 extractor split + # it into 94 and 53 and flagged both against the source. + content="Coverage sits at 94.53 percent across the tests.", + count_sources={"tests": 50}, + ), + CorpusCase( + name="clean_version_number_near_keyword", + label="clean", + # The "10" in "Python 3.10" is a version component, not a module + # count — a nearby source keyword must not turn it into a claim. + content="Requires Python 3.10 to load the python modules.", + count_sources={"python modules": 12}, + ), + CorpusCase( + name="short_label_count_mismatch", + label="hallucinated", + # v0.2.2 dropped label words of length <= 3, so an "api" source + # could never match any claim — silently dead (false negative). + content="The service exposes 42 api endpoints.", + count_sources={"api": 3}, + expected=(ExpectedFinding(FindingKind.COUNT_MISMATCH, "42"),), + ), + CorpusCase( + name="clean_short_label_boundary", + label="clean", + # Short labels match on exact word boundaries only — "api" must not + # match inside "rapid" and drag unrelated numbers into the source. + content="We shipped 42 rapid iterations this quarter.", + count_sources={"api": 3}, + ), + CorpusCase( + name="clean_stopword_in_mixed_label", + label="clean", + # A short word in a MIXED label is usually a stopword: "of" must not + # match ordinary prose and drag unrelated numbers into the source. + # Short-word matching is a fallback for all-short labels only. + content="Only 12 of the widgets remain in the box.", + count_sources={"number of tests": 83}, + ), + CorpusCase( + name="mixed_label_long_word_still_matches", + label="hallucinated", + # The fallback rule must not cost recall: a mixed label still + # matches on its long words. + content="The suite ran 12 tests successfully.", + count_sources={"number of tests": 83}, + expected=(ExpectedFinding(FindingKind.COUNT_MISMATCH, "12"),), + ), + CorpusCase( + name="evasion_flag_in_bash_fence", + label="evasion", + # Flags inside ```bash fences were entirely unchecked in v0.2.2. + content="Enable it:\n```bash\nmytool --nonexistent\n```\n", + help_commands={"mytool": "Options:\n --verbose Be loud\n --help Show help\n"}, + expected=(ExpectedFinding(FindingKind.UNKNOWN_FLAG, "--nonexistent"),), + ), + CorpusCase( + name="clean_link_with_title", + label="clean", + # Markdown title syntax: the title is not part of the path — v0.2.2 + # checked 'docs/a.md "Read me"' for existence and errored. + content='See [the doc](docs/a.md "Read me") for details.', + files=("docs/a.md",), + ), + CorpusCase( + name="clean_link_angle_brackets", + label="clean", + content="See [the doc]() for details.", + files=("docs/a.md",), + ), + CorpusCase( + name="dead_link_with_title_still_flagged", + label="hallucinated", + # Title stripping must not cost recall: a dead path with a title is + # still a dead link. + content='See [the doc](docs/missing.md "Read me").', + expected=(ExpectedFinding(FindingKind.DEAD_LINK, "docs/missing.md"),), + ), ) diff --git a/tests/test_behavioral.py b/tests/test_behavioral.py index a4fe066..18d95ed 100644 --- a/tests/test_behavioral.py +++ b/tests/test_behavioral.py @@ -88,17 +88,18 @@ def test_semantic_disabled_does_not_invoke_judge(): def test_semantic_faithful_produces_no_findings_and_sets_flag(): judge = _FakeJudge(SemanticVerdict(faithful=True)) - ctx = VerifyContext(judge=judge, semantic=True) + ctx = VerifyContext(judge=judge, semantic=True, passages="the source passage") result = verify("the generated answer", ctx) assert result.semantic_ran is True assert [f for f in result.findings if f.kind is FindingKind.SEMANTIC] == [] - # The content is forwarded as the answer. + # The content is the answer; the declared passages are the ground truth. assert judge.calls[0]["answer"] == "the generated answer" + assert judge.calls[0]["passages"] == "the source passage" def test_semantic_unfaithful_emits_one_error_per_issue(): judge = _FakeJudge(SemanticVerdict(faithful=False, issues=["claim A", "claim B"])) - ctx = VerifyContext(judge=judge, semantic=True) + ctx = VerifyContext(judge=judge, semantic=True, passages="the source passage") result = verify("content", ctx) sem = [f for f in result.findings if f.kind is FindingKind.SEMANTIC] assert [f.detail for f in sem] == ["claim A", "claim B"] @@ -112,15 +113,46 @@ def test_semantic_requested_without_judge_warns(): sem = [f for f in result.findings if f.kind is FindingKind.SEMANTIC] assert len(sem) == 1 assert sem[0].severity == "warning" + assert "no judge" in sem[0].detail assert result.semantic_ran is False +def test_semantic_judge_failing_protocol_check_names_the_judge(): + # A judge that IS provided but lacks score() must not be reported as + # "no judge was provided" — the message names the failing object. + class _NotAJudge: + pass + + ctx = VerifyContext(judge=_NotAJudge(), semantic=True, passages="src") + result = verify("content", ctx) + sem = [f for f in result.findings if f.kind is FindingKind.SEMANTIC] + assert len(sem) == 1 + assert sem[0].severity == "warning" + assert "does not satisfy the Judge protocol" in sem[0].detail + assert "_NotAJudge" in sem[0].detail + assert result.semantic_ran is False + + +def test_semantic_without_passages_warns_and_skips_judge(): + # Judging content against itself is vacuous — with no passages the + # judge must not be called at all. + judge = _FakeJudge(SemanticVerdict(faithful=True)) + ctx = VerifyContext(judge=judge, semantic=True) # no passages + result = verify("content", ctx) + sem = [f for f in result.findings if f.kind is FindingKind.SEMANTIC] + assert len(sem) == 1 + assert sem[0].severity == "warning" + assert "passages" in sem[0].detail + assert result.semantic_ran is False + assert judge.calls == [] + + def test_semantic_judge_raising_degrades_to_warning(): class _Raising: def score(self, query, answer, passages): raise RuntimeError("judge down") - ctx = VerifyContext(judge=_Raising(), semantic=True) + ctx = VerifyContext(judge=_Raising(), semantic=True, passages="src") result = verify("content", ctx) sem = [f for f in result.findings if f.kind is FindingKind.SEMANTIC] assert len(sem) == 1 diff --git a/tests/test_regression_author_351.py b/tests/test_regression_author_351.py index 4e00be0..2eb5893 100644 --- a/tests/test_regression_author_351.py +++ b/tests/test_regression_author_351.py @@ -143,8 +143,12 @@ def test_semantic_only_shapes_are_semantic_layer_territory(): ) assert verify(semantic_only, ctx_no_judge).ok is True - # With a judge, both shapes surface as SEMANTIC errors. - ctx = VerifyContext(judge=_StubJudge(), semantic=True) + # With a judge and source passages, both shapes surface as SEMANTIC errors. + ctx = VerifyContext( + judge=_StubJudge(), + semantic=True, + passages="The worker accepts jobs at POST /jobs. Never bind examples to 0.0.0.0.", + ) result = verify(semantic_only, ctx) assert result.semantic_ran is True assert result.ok is False diff --git a/tests/test_semantic.py b/tests/test_semantic.py index a54f2fd..7188f1f 100644 --- a/tests/test_semantic.py +++ b/tests/test_semantic.py @@ -22,6 +22,7 @@ def test_semantic_finding_when_not_faithful(): ctx = VerifyContext( semantic=True, judge=FakeJudge(faithful=False, issues=["insecure example: host=0.0.0.0"]), + passages="Bind the server to a loopback address in examples.", ) result = verify("Some content with host=0.0.0.0", ctx) semantic = [f for f in result.findings if f.kind == FindingKind.SEMANTIC]