Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .reviewgate/lore/review-output-schema-strict.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ id: review-output-schema-strict
status: canon
anchors:
- "src/providers/review-output.ts"
verified_at: 2026-07-10
verified_tree: "3d6c09350b1a2dcede15eb0917e011310c0c327b409895dc00c469491a1a5068"
verified_at: 2026-08-06
verified_tree: "5388991bd6aebe97b8fad3437f141401be49649d18cbc2a772761f301ef94487"
tags: []
---
Why REVIEW_OUTPUT_SCHEMA is shaped the way it is: codex's `--output-schema` runs
Expand Down
816 changes: 816 additions & 0 deletions docs/superpowers/plans/2026-08-05-true-positive-hole.md

Large diffs are not rendered by default.

338 changes: 338 additions & 0 deletions docs/superpowers/specs/2026-08-05-true-positive-hole-design.md

Large diffs are not rendered by default.

19 changes: 17 additions & 2 deletions src/core/aggregator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,7 @@ function memberOf(f: Finding): NonNullable<Finding["members"]>[number] {
// can OR it in (a flagged member merged under an unflagged equal-severity rep must not
// silently lose the flag). Only set when true so members[] stays minimal otherwise.
...(f.demoted_from_critical === true ? { demoted_from_critical: true } : {}),
...(f.anchor_repaired === true ? { anchor_repaired: true } : {}),
};
}

Expand Down Expand Up @@ -529,13 +530,19 @@ export function aggregate(input: AggregateInput): AggregateResult {
const demotedFromCritical =
sample.demoted_from_critical === true ||
members.some((m) => m.demoted_from_critical === true);
// Slice A mirror of the G0 OR above: the repaired finding is often NOT the representative
// (equal severity + ties-keep-first), and losing the marker would hide the mis-anchor in
// exactly the merge the repair made possible.
const anchorRepaired =
sample.anchor_repaired === true || members.some((m) => m.anchor_repaired === true);
deduped.push({
...sample,
details: details.slice(0, 2000),
confirmed_by: reviewers,
consensus,
members,
...(demotedFromCritical ? { demoted_from_critical: true } : {}),
...(anchorRepaired ? { anchor_repaired: true } : {}),
});
}

Expand Down Expand Up @@ -602,6 +609,14 @@ export function aggregate(input: AggregateInput): AggregateResult {
const cv = critic && critSigs.map((s) => critic.get(s)).find((v) => v?.verdict === "likely_fp");
if (cv?.verdict === "likely_fp") {
const isCriticalSecurity = f.severity === "CRITICAL" && touchesSecurityOrCorrectness(f);
// pilot-02 turn 2: the exemption above is keyed to CRITICAL, so a WARN-severity security
// finding was demotable by a single adversarial critic — and WARN→INFO is the one demote
// that crosses the blocking boundary (isBlocking = CRITICAL || WARN). The sibling
// delta-scope pass already exempts security/correctness at ANY severity; mirror that floor
// here. An already-INFO security finding stays droppable, so the critic keeps its
// FP-filtering power where reviewers are noisiest.
const isBlockingSecurity = f.severity === "WARN" && touchesSecurityOrCorrectness(f);
const isSecurityProtected = isCriticalSecurity || isBlockingSecurity;
// A single adversarial critic must not override GROUP agreement. Both
// unanimous AND majority are corroborated consensus — the verdict gate
// treats them identically (warnFail), and the confidence- and reputation-
Expand All @@ -612,11 +627,11 @@ export function aggregate(input: AggregateInput): AggregateResult {
// the critic calls it likely_fp — the dangerous direction is a demoted TRUE positive
// (field report F-005). Tag it so the agent sees WHY it stayed blocking; do NOT set
// critic_verdict (that renders the dismissive "likely FP" badge).
if (!isCriticalSecurity && !isCorroborated && isProtected(f)) {
if (!isSecurityProtected && !isCorroborated && isProtected(f)) {
survivors.push({ ...f, protected_high_precision: true });
continue;
}
if (!isCriticalSecurity && !isCorroborated) {
if (!isSecurityProtected && !isCorroborated) {
const demoted = demoteOneStep(f);
if (demoted.severity === "drop") {
criticDropped.push(f); // INFO likely_fp dropped entirely — keep it attributable
Expand Down
70 changes: 70 additions & 0 deletions src/core/fact-check.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,69 @@ function demote(f: Finding, note: string): Finding {
};
}

/**
* A cited line that does not exist is NOT automatically a fabrication. When the reviewer quoted
* its own evidence and that quote is a real line of the cited file, the reviewer read real code
* and mis-numbered it — mis-anchored, not hallucinated. Demoting that as a hallucination is a
* false accusation against a finding we can PROVE is grounded (pilot-02 turn 2: a 0.90 path
* traversal, quote verbatim at line 26, cited at 67, demoted with "almost certainly
* hallucinated").
*
* `f.evidence_line` is UNTRUSTED reviewer-supplied input — it is never executed or taken on
* trust by itself. It only ever earns a repair by matching, byte-for-byte after normalization,
* real content already read from the working tree; a caller relaxing that normalization or the
* match/identifier-token guards below would let the reviewer's own text decide the outcome.
*
* Returns the finding re-anchored to the quoted line, or null when nothing can be proven — in
* which case the caller demotes exactly as before. This can only ever be reached for an
* out-of-range anchor, so it can never move a valid one.
*
* Multiple matches resolve to the LAST matching occurrence — which, because this only ever runs on
* a citation past EOF (the range check above always fires first), is also the occurrence NEAREST
* the cited line; a tie cannot occur given the range-checked call site. Any matching occurrence is
* a real instance of the reviewer's own quote, so this chooses among facts rather than inventing
* one — and it must be deterministic, because the aggregator's clustering is deliberately
* order-independent and keys on line_start.
*/
function reanchorByEvidence(f: Finding, text: string): Finding | null {
const ev = typeof f.evidence_line === "string" ? f.evidence_line : null;
if (ev === null || ev.length === 0) return null;
const evN = normalizeLine(ev);
if (evN.length === 0) return null; // quote was only whitespace/markers → no signal
// A quote with no identifier-like token at all ("}", "},", "});" — zero [A-Za-z_$] characters
// present) names no code: it matches dozens of lines and proves nothing about WHICH line the
// reviewer read. That rejection holds at ANY length bound, since these quotes contain no
// identifier character to begin with. The {1,} bound (2-character minimum) exists on top of
// that only to exclude a bare single-letter quote ("x", "a") — it still accepts a genuine
// 2-char token, including common JS keywords ("if", "do", "in", "of", "as"). Falling through to
// the demote is fail-safe — it restores exactly the pre-repair behaviour for a quote that
// carries no signal.
if (!/[A-Za-z_$][A-Za-z0-9_$]{1,}/.test(evN)) return null;
const lines = text.split("\n");
const cited = f.line_start - 1;
let best = -1;
for (let i = 0; i < lines.length; i++) {
const raw = lines[i];
if (raw === undefined) continue;
if (normalizeLine(raw) !== evN) continue;
// Strict `<` with an ascending scan keeps the LAST match. Since `cited` is always past EOF
// here, every match index is strictly below it and distance is monotonic in i, so "last
// match" and "nearest match" are the same occurrence — no tie can arise to break.
if (best === -1 || Math.abs(i - cited) < Math.abs(best - cited)) best = i;
}
if (best === -1) return null; // quote matches no line → the fabrication signal stands
const line = best + 1;
const total = lineCount(text);
const note = `\n\n[reviewgate fact-check] the reviewer cited ${f.file}:${f.line_start}, which does not exist (the file has ${total} line${total === 1 ? "" : "s"}), but the evidence it quoted matches line ${line} verbatim — MIS-ANCHORED, not fabricated, so it has been re-anchored there. Treat the defect as real and check line ${line}.`;
return {
...f,
line_start: line,
line_end: line,
anchor_repaired: true,
details: `${f.details.slice(0, 2000 - note.length)}${note}`,
};
}

/**
* Demote findings whose cited file:line provably does not exist (file present but the
* line is out of range / the file is empty). Pure, synchronous, fail-safe.
Expand Down Expand Up @@ -115,6 +178,13 @@ export function validateFindingFacts(
}
const lines = lineCount(text);
if (f.line_start <= lines) return f; // cited line exists → real finding, untouched
// Out of range. Before calling it a fabrication, consult the reviewer's OWN quoted evidence —
// f.evidence_line is UNTRUSTED reviewer-supplied input, so this is a match against real file
// content already read above, never a trust decision made from the quote alone: a quote that
// matches a real line of THIS file proves the reviewer read the code and mis-numbered it,
// which is not what this pass exists to catch.
const repaired = reanchorByEvidence(f, text);
if (repaired !== null) return repaired;
const note = `\n\n[reviewgate fact-check] cited location ${file}:${f.line_start} does not exist in the working tree (file has ${lines} line${lines === 1 ? "" : "s"}) — almost certainly hallucinated; demoted to advisory. Verify before treating as real.`;
return demote(f, note);
});
Expand Down
8 changes: 8 additions & 0 deletions src/core/report-writer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,14 @@ export function findingBadges(f: Finding): string | null {
if (f.deterministic)
badges.push("🔒 deterministic check — fix it (re-runs automatically; not rejectable)");
if (f.fact_invalid) badges.push("🔎 cited location not found — likely hallucinated");
// Anchor repair: the counterpart to the badge above — the cited line was wrong, but the
// reviewer's quoted evidence (carrying an identifier-like token) matched a real line of this
// file, showing the reviewer read real code rather than fabricating one, so it was moved rather
// than demoted. That is weaker than proof the defect itself is real — see finding.ts.
if (f.anchor_repaired)
badges.push(
"⚑ reviewer cited a line that does not exist — re-anchored to the source line it quoted",
);
if (f.grounding_demoted) badges.push("🌫 cited token absent from corpus — likely fabricated");
if (f.hypothetical_demoted)
badges.push(
Expand Down
10 changes: 7 additions & 3 deletions src/providers/review-output.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,13 @@ export const REVIEW_OUTPUT_SCHEMA = {
details: { type: "string" },
confidence: { type: "number" },
// S4 (field report 2026-06-23): the exact source line the finding relies on, verbatim, or
// null if the deciding line/artifact was not provided to the reviewer. RENDER-ONLY: a
// deterministic cross-check (fact-check.ts) badges a CLEAR mismatch vs the working-tree
// line; it never changes severity. Nullable per strict-mode (express optional via type).
// null if the deciding line/artifact was not provided to the reviewer. UNTRUSTED
// reviewer-supplied input: a deterministic cross-check (fact-check.ts) badges a CLEAR
// mismatch vs the working-tree line (render-only, never changes severity), AND — added by
// the true-positive-hole slice — `validateFindingFacts` consults this quote to tell a
// mis-anchored finding (out-of-range line, quote matches real source) from a fabricated
// one, which DOES decide blocking vs advisory. Nullable per strict-mode (express optional
// via type).
evidence_line: { type: ["string", "null"] },
},
},
Expand Down
17 changes: 15 additions & 2 deletions src/schemas/finding.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,8 +122,11 @@ export const FindingSchema = z.object({
// unavailable). NEVER changes severity — purely an ownership tag, unlike foreign_to_session.
session_attributable: z.boolean().optional(),
// S4 (field report 2026-06-23): the exact source line the reviewer self-attests it relied on,
// verbatim (capped). RENDER-ONLY: fact-check badges a CLEAR mismatch vs the working-tree line.
// Never changes severity.
// verbatim (capped). UNTRUSTED reviewer-supplied input. Two consumers: `attestEvidence` badges a
// CLEAR mismatch vs the working-tree line (render-only, never changes severity); and — added by
// the true-positive-hole slice — `validateFindingFacts` (fact-check.ts) consults it to tell a
// MIS-ANCHORED finding (an out-of-range line whose quote matches real source) from a fabricated
// one, which DOES decide blocking vs advisory (see `anchor_repaired`).
evidence_line: z.string().optional(),
// S4: set true (render-only) when the reviewer's quoted evidence_line matches NO line in the cited
// file — a strong signal it reasoned on stale/absent/fabricated context. Advisory badge ONLY; the
Expand Down Expand Up @@ -232,6 +235,12 @@ export const FindingSchema = z.object({
// a fabrication regardless of category, and demoting (vs blocking on a phantom) is
// strictly safer. Fail-safe: any fs uncertainty leaves the finding untouched.
fact_invalid: z.boolean().optional(),
// Slice A (pilot-02 turn 2): set true when the fact-check found the cited line OUT OF RANGE
// but the reviewer's own evidence_line matched a real line of that file — the finding was
// MIS-ANCHORED, not fabricated, so it is re-anchored to the quoted line instead of demoted.
// Severity is untouched; this is a provenance/render marker. Mutually exclusive with
// fact_invalid by construction (the repair returns before the demote).
anchor_repaired: z.boolean().optional(),
// M5 Part B0: per-member provenance of a merged cluster. The aggregator clusters
// findings (possibly different rule_id/category/signature) under one
// representative; this records each member's own signature + trusted base
Expand All @@ -252,6 +261,10 @@ export const FindingSchema = z.object({
// merge can OR-propagate it to the representative (a demoted member merged under
// an unflagged equal-severity representative must not silently lose the flag).
demoted_from_critical: z.boolean().optional(),
// Slice A: per-member mis-anchor provenance, so a repaired member merged under an
// unrepaired equal-severity representative (ties-keep-first) does not silently lose
// the marker — the badge and the pilot count both key on it.
anchor_repaired: z.boolean().optional(),
}),
)
.optional(),
Expand Down
55 changes: 55 additions & 0 deletions tests/unit/aggregator-critic.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -203,3 +203,58 @@ describe("aggregate with critic", () => {
expect(r.dedupedFindings[0]?.consensus).toBe("unanimous");
});
});

describe("critic — security/correctness floor (pilot-02 turn 2)", () => {
// GUARD 7. WITHOUT the floor: INFO + critic_verdict "likely_fp" -> 0 blocking.
// WITH it: WARN + critic_verdict "keep" -> 1 blocking.
it("does not demote a WARN security finding below the blocking boundary", () => {
const f = fin({ signature: "sigSecWarn", severity: "WARN", category: "security" });
const r = aggregate({
findings: [f],
reviewersTotal: 2,
critic: new Map([["sigSecWarn", { verdict: "likely_fp" }]]),
});
expect(r.dedupedFindings[0]?.severity).toBe("WARN");
expect(r.dedupedFindings[0]?.critic_verdict).toBe("keep");
});

it("does not demote a WARN correctness finding below the blocking boundary", () => {
const f = fin({ signature: "sigCorrWarn", severity: "WARN", category: "correctness" });
const r = aggregate({
findings: [f],
reviewersTotal: 2,
critic: new Map([["sigCorrWarn", { verdict: "likely_fp" }]]),
});
expect(r.dedupedFindings[0]?.severity).toBe("WARN");
expect(r.dedupedFindings[0]?.critic_verdict).toBe("keep");
});

// GUARD 8 (passes on current code — MUTATION-CHECKED in Step 3).
// WITH an "exempt at every severity" floor: criticDroppedCount 0.
// WITH the correct WARN floor: criticDroppedCount 1. The critic keeps its FP-filtering
// power exactly where reviewers are noisiest (low-confidence INFO security chatter).
it("still drops an already-INFO security likely_fp", () => {
const f = fin({ signature: "sigSecInfo", severity: "INFO", category: "security" });
const r = aggregate({
findings: [f],
reviewersTotal: 2,
critic: new Map([["sigSecInfo", { verdict: "likely_fp" }]]),
});
expect(r.criticDroppedCount).toBe(1);
expect(r.dedupedFindings.length).toBe(0);
});

// GUARD 9 (passes on current code — MUTATION-CHECKED in Step 3).
// WITH a severity-only floor (all WARN exempt): the finding stays WARN -> 1 blocking.
// WITH the correct category-keyed floor: INFO -> 0 blocking.
it("still demotes a WARN quality finding", () => {
const f = fin({ signature: "sigQualWarn", severity: "WARN", category: "quality" });
const r = aggregate({
findings: [f],
reviewersTotal: 2,
critic: new Map([["sigQualWarn", { verdict: "likely_fp" }]]),
});
expect(r.dedupedFindings[0]?.severity).toBe("INFO");
expect(r.dedupedFindings[0]?.critic_verdict).toBe("likely_fp");
});
});
Loading