diff --git a/README.md b/README.md index ec6f3c0..d174923 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ weeks. None of them are hypothetical. | `hedged_claim` | An agent said a company *"appears to be"* hiring. That became a fact by the time it reached a decision. Hedges are honest; the bug is flattening them downstream. | | `unverified_urls` | Twenty URLs cited, none opened. That is autocomplete, not research. | | `no_output_contract` / `thin_prompt` | The failure that is **yours, not the model's**. A task with no definition of done cannot be verified, because you never really asked the question. | +| `unquantified` | A precise count/list prompt answered with *"several hot paths worth investigating"* and no digits, bullets, or table rows. | | `slow_run` | A subagent running 25 minutes is usually looping or retrying. | Run against the session that motivated the tool, it flags **36 of 152 runs**: 7 agents that died on diff --git a/agentrace/checks.py b/agentrace/checks.py index bf3c66a..d8eba15 100644 --- a/agentrace/checks.py +++ b/agentrace/checks.py @@ -202,6 +202,47 @@ def check_prompt_hygiene(run: AgentRun) -> list[Finding]: return out + +def check_unquantified(run: AgentRun) -> list[Finding]: + """A counting prompt answered with vague quantity and no countable output. + + The real case: "report every failing test as node ids" came back as "several hot + paths worth investigating" — zero digits, bullets, or table rows. + """ + asks_to_count = re.search( + r"\b(?:count|counts|how many|number of|enumerate|enumerat(?:e|ing)|" + r"list (?:every|all|each)|report every|every \w+ as)\b", + run.prompt, + re.I, + ) + if not asks_to_count: + return [] + + result = run.result + has_digit = re.search(r"\d", result) is not None + has_bullet = re.search(r"(?m)^\s*(?:[-*+]|\d+[.)])\s+\S", result) is not None + has_table_row = re.search(r"(?m)^\s*\|.+\|\s*$", result) is not None + if has_digit or has_bullet or has_table_row: + return [] + + vague = re.search( + r"\b(?:several|some|a few|a number of|various|numerous|many|multiple)\b", + result, + re.I, + ) + if not vague: + return [] + + return [ + Finding( + "unquantified", + "low", + "Prompt asked to count, list, or enumerate, but the result uses a vague quantifier and has no digits, bullets, or table rows.", + _context(result, vague.start()), + ) + ] + + def check_runaway(run: AgentRun, slow_s: float = 900.0) -> list[Finding]: """Very long runs. @@ -227,6 +268,7 @@ def check_runaway(run: AgentRun, slow_s: float = 900.0) -> list[Finding]: check_absence_as_evidence, check_url_without_verification, check_prompt_hygiene, + check_unquantified, check_runaway, ] diff --git a/tests/test_agentrace.py b/tests/test_agentrace.py index 48dca39..a97503c 100644 --- a/tests/test_agentrace.py +++ b/tests/test_agentrace.py @@ -261,6 +261,26 @@ def test_prompt_with_output_contract_is_not_flagged(): assert "no_output_contract" not in _codes(r) + +def test_unquantified_flags_vague_answer_to_counting_prompt(): + """'report every failing test as node ids' answered with 'several hot paths'.""" + r = _run( + prompt="Run the suite and report every failing test as node ids with its assertion message.", + result="There are several hot paths worth investigating. " + "x" * 200, + ) + findings = [f for f in analyse(r) if f.check == "unquantified"] + assert findings and findings[0].severity == "low" + + +def test_unquantified_clean_when_result_has_counts(): + r = _run( + prompt="Run the suite and report every failing test as node ids with its assertion message.", + result="3 failing: tests/test_a.py::test_one, tests/test_b.py::test_two, tests/test_c.py::test_three. " + + "x" * 200, + ) + assert "unquantified" not in _codes(r) + + def test_slow_run_is_flagged(): r = _run( started_at=datetime(2026, 7, 16, 12, 0, tzinfo=timezone.utc),