Add unquantified check for vague answers to counting prompts

README.md

@@ -55,6 +55,7 @@ weeks. None of them are hypothetical.
 | `hedged_claim` | An agent said a company *"appears to be"* hiring. That became a fact by the time it reached a decision. Hedges are honest; the bug is flattening them downstream. |
 | `unverified_urls` | Twenty URLs cited, none opened. That is autocomplete, not research. |
 | `no_output_contract` / `thin_prompt` | The failure that is **yours, not the model's**. A task with no definition of done cannot be verified, because you never really asked the question. |
+| `unquantified` | A precise count/list prompt answered with *"several hot paths worth investigating"* and no digits, bullets, or table rows. |
 | `slow_run` | A subagent running 25 minutes is usually looping or retrying. |
 
 Run against the session that motivated the tool, it flags **36 of 152 runs**: 7 agents that died on

agentrace/checks.py

@@ -202,6 +202,47 @@ def check_prompt_hygiene(run: AgentRun) -> list[Finding]:
     return out
 
 
+
+def check_unquantified(run: AgentRun) -> list[Finding]:
+    """A counting prompt answered with vague quantity and no countable output.
+
+    The real case: "report every failing test as node ids" came back as "several hot
+    paths worth investigating" — zero digits, bullets, or table rows.
+    """
+    asks_to_count = re.search(
+        r"\b(?:count|counts|how many|number of|enumerate|enumerat(?:e|ing)|"
+        r"list (?:every|all|each)|report every|every \w+ as)\b",
+        run.prompt,
+        re.I,
+    )
+    if not asks_to_count:
+        return []
+
+    result = run.result
+    has_digit = re.search(r"\d", result) is not None
+    has_bullet = re.search(r"(?m)^\s*(?:[-*+]|\d+[.)])\s+\S", result) is not None
+    has_table_row = re.search(r"(?m)^\s*\|.+\|\s*$", result) is not None
+    if has_digit or has_bullet or has_table_row:
+        return []
+
+    vague = re.search(
+        r"\b(?:several|some|a few|a number of|various|numerous|many|multiple)\b",
+        result,
+        re.I,
+    )
+    if not vague:
+        return []
+
+    return [
+        Finding(
+            "unquantified",
+            "low",
+            "Prompt asked to count, list, or enumerate, but the result uses a vague quantifier and has no digits, bullets, or table rows.",
+            _context(result, vague.start()),
+        )
+    ]
+
+
 def check_runaway(run: AgentRun, slow_s: float = 900.0) -> list[Finding]:
     """Very long runs.
 
@@ -227,6 +268,7 @@ def check_runaway(run: AgentRun, slow_s: float = 900.0) -> list[Finding]:
     check_absence_as_evidence,
     check_url_without_verification,
     check_prompt_hygiene,
+    check_unquantified,
     check_runaway,
 ]
 

tests/test_agentrace.py

@@ -261,6 +261,26 @@ def test_prompt_with_output_contract_is_not_flagged():
     assert "no_output_contract" not in _codes(r)
 
 
+
+def test_unquantified_flags_vague_answer_to_counting_prompt():
+    """'report every failing test as node ids' answered with 'several hot paths'."""
+    r = _run(
+        prompt="Run the suite and report every failing test as node ids with its assertion message.",
+        result="There are several hot paths worth investigating. " + "x" * 200,
+    )
+    findings = [f for f in analyse(r) if f.check == "unquantified"]
+    assert findings and findings[0].severity == "low"
+
+
+def test_unquantified_clean_when_result_has_counts():
+    r = _run(
+        prompt="Run the suite and report every failing test as node ids with its assertion message.",
+        result="3 failing: tests/test_a.py::test_one, tests/test_b.py::test_two, tests/test_c.py::test_three. "
+        + "x" * 200,
+    )
+    assert "unquantified" not in _codes(r)
+
+
 def test_slow_run_is_flagged():
     r = _run(
         started_at=datetime(2026, 7, 16, 12, 0, tzinfo=timezone.utc),