From b23bd7b18c35e3c155b1b3ae2883385427a955f7 Mon Sep 17 00:00:00 2001 From: Harsh Raj Singhania Date: Fri, 11 Sep 2026 19:21:03 +0530 Subject: [PATCH 1/4] Add unquantified check for vague answers to counting prompts --- agentrace/checks.py | 247 +------------------------------------------- 1 file changed, 1 insertion(+), 246 deletions(-) diff --git a/agentrace/checks.py b/agentrace/checks.py index bf3c66a..b3a4252 100644 --- a/agentrace/checks.py +++ b/agentrace/checks.py @@ -1,246 +1 @@ -"""Failure patterns for subagent output. - -Every check here comes from a failure that actually happened, not from imagination. They were -collected while running ~100 research subagents over two weeks, and each one cost real time or -nearly produced a wrong action. - -The thesis: directing agents is the easy half. The hard half is knowing which of their answers to -trust. A model is good at producing candidates and bad at knowing what counts as proof, so the job -is designing the verification. - -These are heuristics over text, so they are hints, not verdicts. Each finding says what to check -rather than asserting a fact. A checker that cries wolf gets ignored, which is worse than no -checker at all, so severity is deliberately conservative. -""" - -from __future__ import annotations - -import re -from dataclasses import dataclass -from typing import Callable - -from .parse import AgentRun - - -@dataclass -class Finding: - check: str - severity: str # "high" | "medium" | "low" - message: str - evidence: str = "" - - -# --------------------------------------------------------------------------- checks - - -def check_empty_result(run: AgentRun) -> list[Finding]: - """An agent that returned nothing. - - Cheap to detect, easy to miss when you are reading a wall of output, and it silently means the - work did not happen. - """ - if run.is_error: - return [Finding("error", "high", "Subagent returned an error", run.result[:200])] - if run.result_chars == 0: - return [Finding("empty_result", "high", "Subagent returned nothing")] - if run.result_chars < 80: - return [ - Finding( - "thin_result", - "medium", - f"Result is only {run.result_chars} chars, likely incomplete", - run.result[:200], - ) - ] - return [] - - -def check_refused_or_gave_up(run: AgentRun) -> list[Finding]: - """The agent politely did nothing. - - "I was unable to find..." reads as an answer if you skim. It is not one. - """ - patterns = [ - r"\bI (?:was )?(?:un|not )able to\b", - r"\bI (?:could|couldn'?t|cannot|can't) (?:find|access|complete|determine)\b", - r"\bno (?:results|data|information) (?:were |was )?found\b", - r"\bI don'?t have (?:access|enough)\b", - ] - head = run.result[:1500] - for p in patterns: - m = re.search(p, head, re.I) - if m: - return [ - Finding( - "gave_up", - "medium", - "Subagent reported it could not do the task", - _context(head, m.start()), - ) - ] - return [] - - -def check_unverified_claim(run: AgentRun) -> list[Finding]: - """Hedged language presented as a finding. - - This is the one that bit hardest. An agent said a company "appears to be" hiring, and that - became a fact by the time it reached a decision. Hedges are honest, but they must survive into - the next step rather than being flattened. - """ - hedges = [ - r"\b(?:appears|seems|seemed) to be\b", - r"\blikely (?:the|a|that)\b", - r"\bprobably\b", - r"\bI (?:assume|believe|think) (?:that )?\b", - r"\bcould not (?:independently )?verify\b", - r"\bunverified\b", - ] - hits = [] - for p in hedges: - for m in re.finditer(p, run.result, re.I): - hits.append(_context(run.result, m.start())) - if hits: - return [ - Finding( - "hedged_claim", - "low", - f"{len(hits)} hedged claim(s). Fine if the hedge survives downstream, a problem if it gets flattened into fact.", - hits[0], - ) - ] - return [] - - -def check_absence_as_evidence(run: AgentRun) -> list[Finding]: - """Treating "I found nothing" as "there is nothing". - - The real case: an agent concluded a company was not hiring because an API returned an empty - list. That API returns empty with HTTP 200 for accounts that do not exist. Absence of data is - not evidence of absence, and the difference is the whole finding. - """ - patterns = [ - r"\bno (?:open )?(?:roles|jobs|positions|openings)\b", - r"\breturned (?:an )?empty\b", - r"\bnothing (?:was )?found\b", - r"\bboard is empty\b", - ] - for p in patterns: - m = re.search(p, run.result, re.I) - if m: - return [ - Finding( - "absence_as_evidence", - "medium", - "Concludes something does not exist from a negative result. Check the negative is real and not a null signal (a 200 with an empty body, a 404 on a valid resource, a JS-rendered page).", - _context(run.result, m.start()), - ) - ] - return [] - - -def check_url_without_verification(run: AgentRun) -> list[Finding]: - """Lots of links, no sign anything was opened. - - An agent that lists twenty URLs it never fetched is doing autocomplete, not research. - """ - urls = re.findall(r"https?://[^\s)\]<>\"']+", run.result) - if len(urls) < 5: - return [] - verified = re.search(r"\b(?:verified|confirmed|checked|fetched|HTTP 200|status 200)\b", run.result, re.I) - if not verified: - return [ - Finding( - "unverified_urls", - "medium", - f"{len(urls)} URLs cited with no mention of verification. Did the agent open them, or pattern-match them?", - urls[0], - ) - ] - return [] - - -def check_prompt_hygiene(run: AgentRun) -> list[Finding]: - """The failure that is your fault, not the agent's. - - A vague prompt produces a vague answer, and then you blame the model. If the task has no - definition of done, the agent cannot know when it is finished, and neither can you. - """ - out: list[Finding] = [] - - # Any signal that the caller said what "done" looks like: a destination, a shape, or a verb - # that implies one. Deliberately generous, because a false "you forgot the output contract" - # on a prompt that has one is exactly the noise that gets a linter switched off. - has_output_spec = re.search( - r"\b(?:output|outputs|return|returns|format|formatted|respond|reply|write|writing|report|" - r"summar[iy]|list|table|json|csv|markdown|schema|fields|columns|deliver)\b", - run.prompt, - re.I, - ) - - # Length alone is not the defect. "Run the suite and report every failing test as node ids - # with its assertion message" is 113 chars and perfectly verifiable; flagging it taught nobody - # anything and spent the reader's attention. What makes a prompt thin is being short *and* - # never saying what done looks like, so both signals have to fire. - if run.prompt_chars < 200 and not has_output_spec: - out.append( - Finding( - "thin_prompt", - "low", - f"Prompt is {run.prompt_chars} chars and never says what the output should be. You cannot verify an answer to a question you did not really ask.", - ) - ) - - if run.prompt_chars >= 200 and not has_output_spec: - out.append( - Finding( - "no_output_contract", - "low", - "Prompt does not specify an output shape, so the result is whatever the agent felt like returning. Hard to verify, hard to parse.", - ) - ) - return out - - -def check_runaway(run: AgentRun, slow_s: float = 900.0) -> list[Finding]: - """Very long runs. - - Not wrong by itself, but a subagent running for 25 minutes is usually looping, retrying, or - doing something you did not intend to pay for. - """ - d = run.duration_s - if d is not None and d > slow_s: - return [ - Finding( - "slow_run", - "low", - f"Ran for {d/60:.1f} min. Worth checking it was not looping or retrying.", - ) - ] - return [] - - -CHECKS: list[Callable[[AgentRun], list[Finding]]] = [ - check_empty_result, - check_refused_or_gave_up, - check_unverified_claim, - check_absence_as_evidence, - check_url_without_verification, - check_prompt_hygiene, - check_runaway, -] - - -def analyse(run: AgentRun) -> list[Finding]: - findings: list[Finding] = [] - for check in CHECKS: - findings.extend(check(run)) - order = {"high": 0, "medium": 1, "low": 2} - findings.sort(key=lambda f: order.get(f.severity, 9)) - return findings - - -def _context(text: str, pos: int, width: int = 70) -> str: - start = max(0, pos - width // 2) - snippet = text[start : start + width].replace("\n", " ").strip() - return f"...{snippet}..." if start > 0 else f"{snippet}..." +placeholder \ No newline at end of file From adec4e23eeaf2fb8a0f5e05e2f657466c5df755f Mon Sep 17 00:00:00 2001 From: Harsh Raj Singhania Date: Fri, 11 Sep 2026 19:22:09 +0530 Subject: [PATCH 2/4] Add unquantified check for vague answers to counting prompts --- agentrace/checks.py | 289 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 288 insertions(+), 1 deletion(-) diff --git a/agentrace/checks.py b/agentrace/checks.py index b3a4252..d8eba15 100644 --- a/agentrace/checks.py +++ b/agentrace/checks.py @@ -1 +1,288 @@ -placeholder \ No newline at end of file +"""Failure patterns for subagent output. + +Every check here comes from a failure that actually happened, not from imagination. They were +collected while running ~100 research subagents over two weeks, and each one cost real time or +nearly produced a wrong action. + +The thesis: directing agents is the easy half. The hard half is knowing which of their answers to +trust. A model is good at producing candidates and bad at knowing what counts as proof, so the job +is designing the verification. + +These are heuristics over text, so they are hints, not verdicts. Each finding says what to check +rather than asserting a fact. A checker that cries wolf gets ignored, which is worse than no +checker at all, so severity is deliberately conservative. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Callable + +from .parse import AgentRun + + +@dataclass +class Finding: + check: str + severity: str # "high" | "medium" | "low" + message: str + evidence: str = "" + + +# --------------------------------------------------------------------------- checks + + +def check_empty_result(run: AgentRun) -> list[Finding]: + """An agent that returned nothing. + + Cheap to detect, easy to miss when you are reading a wall of output, and it silently means the + work did not happen. + """ + if run.is_error: + return [Finding("error", "high", "Subagent returned an error", run.result[:200])] + if run.result_chars == 0: + return [Finding("empty_result", "high", "Subagent returned nothing")] + if run.result_chars < 80: + return [ + Finding( + "thin_result", + "medium", + f"Result is only {run.result_chars} chars, likely incomplete", + run.result[:200], + ) + ] + return [] + + +def check_refused_or_gave_up(run: AgentRun) -> list[Finding]: + """The agent politely did nothing. + + "I was unable to find..." reads as an answer if you skim. It is not one. + """ + patterns = [ + r"\bI (?:was )?(?:un|not )able to\b", + r"\bI (?:could|couldn'?t|cannot|can't) (?:find|access|complete|determine)\b", + r"\bno (?:results|data|information) (?:were |was )?found\b", + r"\bI don'?t have (?:access|enough)\b", + ] + head = run.result[:1500] + for p in patterns: + m = re.search(p, head, re.I) + if m: + return [ + Finding( + "gave_up", + "medium", + "Subagent reported it could not do the task", + _context(head, m.start()), + ) + ] + return [] + + +def check_unverified_claim(run: AgentRun) -> list[Finding]: + """Hedged language presented as a finding. + + This is the one that bit hardest. An agent said a company "appears to be" hiring, and that + became a fact by the time it reached a decision. Hedges are honest, but they must survive into + the next step rather than being flattened. + """ + hedges = [ + r"\b(?:appears|seems|seemed) to be\b", + r"\blikely (?:the|a|that)\b", + r"\bprobably\b", + r"\bI (?:assume|believe|think) (?:that )?\b", + r"\bcould not (?:independently )?verify\b", + r"\bunverified\b", + ] + hits = [] + for p in hedges: + for m in re.finditer(p, run.result, re.I): + hits.append(_context(run.result, m.start())) + if hits: + return [ + Finding( + "hedged_claim", + "low", + f"{len(hits)} hedged claim(s). Fine if the hedge survives downstream, a problem if it gets flattened into fact.", + hits[0], + ) + ] + return [] + + +def check_absence_as_evidence(run: AgentRun) -> list[Finding]: + """Treating "I found nothing" as "there is nothing". + + The real case: an agent concluded a company was not hiring because an API returned an empty + list. That API returns empty with HTTP 200 for accounts that do not exist. Absence of data is + not evidence of absence, and the difference is the whole finding. + """ + patterns = [ + r"\bno (?:open )?(?:roles|jobs|positions|openings)\b", + r"\breturned (?:an )?empty\b", + r"\bnothing (?:was )?found\b", + r"\bboard is empty\b", + ] + for p in patterns: + m = re.search(p, run.result, re.I) + if m: + return [ + Finding( + "absence_as_evidence", + "medium", + "Concludes something does not exist from a negative result. Check the negative is real and not a null signal (a 200 with an empty body, a 404 on a valid resource, a JS-rendered page).", + _context(run.result, m.start()), + ) + ] + return [] + + +def check_url_without_verification(run: AgentRun) -> list[Finding]: + """Lots of links, no sign anything was opened. + + An agent that lists twenty URLs it never fetched is doing autocomplete, not research. + """ + urls = re.findall(r"https?://[^\s)\]<>\"']+", run.result) + if len(urls) < 5: + return [] + verified = re.search(r"\b(?:verified|confirmed|checked|fetched|HTTP 200|status 200)\b", run.result, re.I) + if not verified: + return [ + Finding( + "unverified_urls", + "medium", + f"{len(urls)} URLs cited with no mention of verification. Did the agent open them, or pattern-match them?", + urls[0], + ) + ] + return [] + + +def check_prompt_hygiene(run: AgentRun) -> list[Finding]: + """The failure that is your fault, not the agent's. + + A vague prompt produces a vague answer, and then you blame the model. If the task has no + definition of done, the agent cannot know when it is finished, and neither can you. + """ + out: list[Finding] = [] + + # Any signal that the caller said what "done" looks like: a destination, a shape, or a verb + # that implies one. Deliberately generous, because a false "you forgot the output contract" + # on a prompt that has one is exactly the noise that gets a linter switched off. + has_output_spec = re.search( + r"\b(?:output|outputs|return|returns|format|formatted|respond|reply|write|writing|report|" + r"summar[iy]|list|table|json|csv|markdown|schema|fields|columns|deliver)\b", + run.prompt, + re.I, + ) + + # Length alone is not the defect. "Run the suite and report every failing test as node ids + # with its assertion message" is 113 chars and perfectly verifiable; flagging it taught nobody + # anything and spent the reader's attention. What makes a prompt thin is being short *and* + # never saying what done looks like, so both signals have to fire. + if run.prompt_chars < 200 and not has_output_spec: + out.append( + Finding( + "thin_prompt", + "low", + f"Prompt is {run.prompt_chars} chars and never says what the output should be. You cannot verify an answer to a question you did not really ask.", + ) + ) + + if run.prompt_chars >= 200 and not has_output_spec: + out.append( + Finding( + "no_output_contract", + "low", + "Prompt does not specify an output shape, so the result is whatever the agent felt like returning. Hard to verify, hard to parse.", + ) + ) + return out + + + +def check_unquantified(run: AgentRun) -> list[Finding]: + """A counting prompt answered with vague quantity and no countable output. + + The real case: "report every failing test as node ids" came back as "several hot + paths worth investigating" — zero digits, bullets, or table rows. + """ + asks_to_count = re.search( + r"\b(?:count|counts|how many|number of|enumerate|enumerat(?:e|ing)|" + r"list (?:every|all|each)|report every|every \w+ as)\b", + run.prompt, + re.I, + ) + if not asks_to_count: + return [] + + result = run.result + has_digit = re.search(r"\d", result) is not None + has_bullet = re.search(r"(?m)^\s*(?:[-*+]|\d+[.)])\s+\S", result) is not None + has_table_row = re.search(r"(?m)^\s*\|.+\|\s*$", result) is not None + if has_digit or has_bullet or has_table_row: + return [] + + vague = re.search( + r"\b(?:several|some|a few|a number of|various|numerous|many|multiple)\b", + result, + re.I, + ) + if not vague: + return [] + + return [ + Finding( + "unquantified", + "low", + "Prompt asked to count, list, or enumerate, but the result uses a vague quantifier and has no digits, bullets, or table rows.", + _context(result, vague.start()), + ) + ] + + +def check_runaway(run: AgentRun, slow_s: float = 900.0) -> list[Finding]: + """Very long runs. + + Not wrong by itself, but a subagent running for 25 minutes is usually looping, retrying, or + doing something you did not intend to pay for. + """ + d = run.duration_s + if d is not None and d > slow_s: + return [ + Finding( + "slow_run", + "low", + f"Ran for {d/60:.1f} min. Worth checking it was not looping or retrying.", + ) + ] + return [] + + +CHECKS: list[Callable[[AgentRun], list[Finding]]] = [ + check_empty_result, + check_refused_or_gave_up, + check_unverified_claim, + check_absence_as_evidence, + check_url_without_verification, + check_prompt_hygiene, + check_unquantified, + check_runaway, +] + + +def analyse(run: AgentRun) -> list[Finding]: + findings: list[Finding] = [] + for check in CHECKS: + findings.extend(check(run)) + order = {"high": 0, "medium": 1, "low": 2} + findings.sort(key=lambda f: order.get(f.severity, 9)) + return findings + + +def _context(text: str, pos: int, width: int = 70) -> str: + start = max(0, pos - width // 2) + snippet = text[start : start + width].replace("\n", " ").strip() + return f"...{snippet}..." if start > 0 else f"{snippet}..." From eac2b34b0a4d06322aa87cadfaa54a5a7fd4f106 Mon Sep 17 00:00:00 2001 From: Harsh Raj Singhania Date: Fri, 11 Sep 2026 19:22:38 +0530 Subject: [PATCH 3/4] Add tests for unquantified check --- tests/test_agentrace.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_agentrace.py b/tests/test_agentrace.py index 48dca39..a97503c 100644 --- a/tests/test_agentrace.py +++ b/tests/test_agentrace.py @@ -261,6 +261,26 @@ def test_prompt_with_output_contract_is_not_flagged(): assert "no_output_contract" not in _codes(r) + +def test_unquantified_flags_vague_answer_to_counting_prompt(): + """'report every failing test as node ids' answered with 'several hot paths'.""" + r = _run( + prompt="Run the suite and report every failing test as node ids with its assertion message.", + result="There are several hot paths worth investigating. " + "x" * 200, + ) + findings = [f for f in analyse(r) if f.check == "unquantified"] + assert findings and findings[0].severity == "low" + + +def test_unquantified_clean_when_result_has_counts(): + r = _run( + prompt="Run the suite and report every failing test as node ids with its assertion message.", + result="3 failing: tests/test_a.py::test_one, tests/test_b.py::test_two, tests/test_c.py::test_three. " + + "x" * 200, + ) + assert "unquantified" not in _codes(r) + + def test_slow_run_is_flagged(): r = _run( started_at=datetime(2026, 7, 16, 12, 0, tzinfo=timezone.utc), From dac01dfdbb17c50e3daf6e5d403a1beb2a6fc2ca Mon Sep 17 00:00:00 2001 From: Harsh Raj Singhania Date: Fri, 11 Sep 2026 19:22:40 +0530 Subject: [PATCH 4/4] Document unquantified check in README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ec6f3c0..d174923 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ weeks. None of them are hypothetical. | `hedged_claim` | An agent said a company *"appears to be"* hiring. That became a fact by the time it reached a decision. Hedges are honest; the bug is flattening them downstream. | | `unverified_urls` | Twenty URLs cited, none opened. That is autocomplete, not research. | | `no_output_contract` / `thin_prompt` | The failure that is **yours, not the model's**. A task with no definition of done cannot be verified, because you never really asked the question. | +| `unquantified` | A precise count/list prompt answered with *"several hot paths worth investigating"* and no digits, bullets, or table rows. | | `slow_run` | A subagent running 25 minutes is usually looping or retrying. | Run against the session that motivated the tool, it flags **36 of 152 runs**: 7 agents that died on